Update template

[noissue]

.ci/ansible/Containerfile.j2

@@ -2,18 +2,15 @@ FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }
 
 # Add source directories to container
 {% for item in plugins %}
-{% if item.source.startswith("./") %}
-ADD {{ item.source }} {{ item.source }}
+{% if item.source.startswith("./") or item.ci_requirements | default(false) %}
+ADD ./{{ item.name }} ./{{ item.name }}
 {% endif %}
 {% endfor %}
 
 # Install python packages
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
-RUN pip3 install \
-{%- if stream_test | default(false) -%}
-{{ " " }}django-storages[sftp]
-{%- endif -%}
+RUN pip3 install
 {%- if s3_test | default(false) -%}
 {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
 {%- endif -%}
@@ -27,7 +24,10 @@ RUN pip3 install \
 {%- if item.name == "pulp-certguard" -%}
 {{ " " }}python-dateutil rhsm
 {%- endif -%}
-{{ " " }}"{{ item.source }}"
+{{ " " }}{{ item.source }}
+{%- if item.ci_requirements | default(false) -%}
+{{ " " }}-r ./{{ item.name }}/ci_requirements.txt
+{%- endif -%}
 {%- endfor %}
 
 USER pulp:pulp

.ci/ansible/settings.py.j2

@@ -26,18 +26,6 @@ API_ROOT = {{ api_root | repr }}
 {% endfor %}
 {% endif %}
 
-{% if stream_test | default(false) -%}
-REDIRECT_TO_OBJECT_STORAGE = False
-DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.PulpSFTPStorage"
-MEDIA_ROOT = ""
-SFTP_STORAGE_HOST = "ci-sftp"
-SFTP_STORAGE_ROOT = "/storage/"
-SFTP_STORAGE_PARAMS = {
-    "username": "foo",
-    "key_filename": "/keys/id_ed25519",
-}
-{%- endif %}
-
 {% if s3_test | default(false) %}
 DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 MEDIA_ROOT = ""

.ci/ansible/start_container.yaml

@@ -15,14 +15,6 @@
         - ssh
         - ~/.config/pulp_smash
 
-    - name: Generate an OpenSSH keypair
-      community.crypto.openssh_keypair:
-        path: ssh/id_ed25519
-        type: ed25519
-        owner: 700  # pulp in the container
-      become: true
-      when: stream_test | default(false)
-
     - name: "Generate Pulp Settings"
       template:
         src: settings.py.j2

.ci/scripts/calc_deps_lowerbounds.py

@@ -8,13 +8,6 @@
 from packaging.requirements import Requirement
 
 
-# We install these from source, but it would be really handy if we could test pulpcore
-# compatibility that way too.
-EXCEPTIONS = [
-    "pulpcore",
-]
-
-
 def main():
     """Calculate the lower bound of dependencies where possible."""
     with open("requirements.txt") as req_file:
@@ -24,16 +17,17 @@ def main():
             except ValueError:
                 print(line.strip())
             else:
-                if requirement.name.replace("-", "_") in EXCEPTIONS:
-                    print(line.strip())
+                for spec in requirement.specifier:
+                    if spec.operator == ">=":
+                        if requirement.name == "pulpcore":
+                            operator = "~="
+                        else:
+                            operator = "=="
+                        min_version = str(spec)[2:]
+                        print(f"{requirement.name}{operator}{min_version}")
+                        break
                 else:
-                    for spec in requirement.specifier:
-                        if spec.operator == ">=":
-                            min_version = str(spec)[2:]
-                            print(f"{requirement.name}=={min_version}")
-                            break
-                    else:
-                        print(line.strip())
+                    print(line.strip())
 
 
 if __name__ == "__main__":

.ci/scripts/check_release.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_rpm' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import argparse
+import re
+import os
+import yaml
+from tempfile import TemporaryDirectory
+from packaging.version import Version
+from git import Repo
+
+UPSTREAM_REMOTE = "https://github.com/pulp/pulp_rpm.git"
+DEFAULT_BRANCH = "main"
+RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$"
+Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"]
+Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"]
+
+
+def main():
+    """Check which branches need a release."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--branches",
+        default="supported",
+        help="A comma separated list of branches to check for releases. Can also use keyword: "
+        "'supported'. Defaults to 'supported', see `ci_update_branches` in "
+        "`plugin_template.yml`.",
+    )
+    opts = parser.parse_args()
+
+    with TemporaryDirectory() as d:
+        # Clone from upstream to ensure we have updated branches & main
+        repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none")
+        heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")]
+        available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)]
+        available_branches.sort(key=lambda ver: Version(ver))
+        available_branches.append(DEFAULT_BRANCH)
+
+        branches = opts.branches
+        if branches == "supported":
+            with open(f"{d}/template_config.yml", mode="r") as f:
+                tc = yaml.safe_load(f)
+                branches = tc["ci_update_branches"]
+            branches.append(DEFAULT_BRANCH)
+        else:
+            branches = branches.split(",")
+
+        if diff := set(branches) - set(available_branches):
+            print(f"Supplied branches contains non-existent branches! {diff}")
+            exit(1)
+
+        print(f"Checking for releases on branches: {branches}")
+
+        releases = []
+        for branch in branches:
+            if branch != DEFAULT_BRANCH:
+                # Check if a Z release is needed
+                changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/")
+                z_release = False
+                for change in changes.split("\n"):
+                    # Check each changelog file to make sure everything checks out
+                    _, ext = os.path.splitext(change)
+                    if ext in Y_CHANGELOG_EXTS:
+                        print(
+                            f"Warning: A non-backported changelog ({change}) is present in the "
+                            f"{branch} release branch!"
+                        )
+                    elif ext in Z_CHANGELOG_EXTS:
+                        z_release = True
+                if z_release:
+                    # Blobless clone does not have file contents for Z branches,
+                    # check commit message for last Z bump
+                    git_branch = f"origin/{branch}"
+                    next_version = repo.git.log(
+                        "--oneline", "--grep=Bump to", "-n 1", git_branch, "--", ".bumpversion.cfg"
+                    ).split("to")[-1]
+                    next_version = Version(next_version)
+                    print(
+                        f"A Z-release is needed for {branch}, "
+                        f"New Version: {next_version.base_version}"
+                    )
+                    releases.append(next_version)
+            else:
+                # Check if a Y release is needed
+                changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/")
+                for change in changes.split("\n"):
+                    _, ext = os.path.splitext(change)
+                    if ext in Y_CHANGELOG_EXTS:
+                        # We don't put Y release bumps in the commit message, check file instead
+                        # The 'current_version' is always the next version to release
+                        next_version = repo.git.grep(
+                            "current_version", DEFAULT_BRANCH, "--", ".bumpversion.cfg"
+                        ).split("=")[-1]
+                        next_version = Version(next_version)
+                        print(
+                            f"A new Y-release is needed! New Version: {next_version.base_version}"
+                        )
+                        releases.append(next_version)
+                        break
+
+        if len(releases) == 0:
+            print("No new releases to perform.")
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/check_requirements.py

@@ -0,0 +1,66 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_rpm' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import warnings
+from pkg_resources import Requirement
+
+
+CHECK_MATRIX = [
+    ("requirements.txt", True, True, True),
+    ("dev_requirements.txt", False, True, False),
+    ("ci_requirements.txt", False, True, True),
+    ("doc_requirements.txt", False, True, False),
+    ("lint_requirements.txt", False, True, True),
+    ("unittest_requirements.txt", False, True, True),
+    ("functest_requirements.txt", False, True, True),
+    ("clitest_requirements.txt", False, True, True),
+]
+
+errors = []
+
+for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX:
+    try:
+        with open(filename, "r") as fd:
+            for nr, line in enumerate(fd.readlines()):
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                try:
+                    req = Requirement.parse(line)
+                except ValueError:
+                    if line.startswith("git+"):
+                        # The single exception...
+                        if "pulp-smash" not in line:
+                            errors.append(f"{filename}:{nr}: Invalid source requirement: {line}")
+                    elif line.startswith("-r "):
+                        if check_r:
+                            errors.append(f"{filename}:{nr}: Invalid deferred requirement: {line}")
+                    else:
+                        errors.append(f"{filename}:{nr}: Unreadable requirement {line}")
+                else:
+                    if check_prereleases and req.specifier.prereleases:
+                        # Do not even think about begging for more exceptions!
+                        if (
+                            not req.name.startswith("opentelemetry")
+                            and req.name != "pulp-rpm-client"
+                        ):
+                            errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.")
+                    ops = [op for op, ver in req.specs]
+                    spec = str(req.specs)
+                    if "~=" in ops:
+                        warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!")
+                    elif "<" not in ops and "<=" not in ops and "==" not in ops:
+                        if check_upperbound:
+                            errors.append(f"{filename}:{nr}: Upper bound missing in {line}.")
+    except FileNotFoundError:
+        # skip this test for plugins that don't use this requirements.txt
+        pass
+
+if errors:
+    print("Dependency issues found:")
+    print("\n".join(errors))
+    exit(1)

.ci/scripts/collect_changes.py

@@ -0,0 +1,101 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_rpm' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import itertools
+import os
+import re
+
+import toml
+from git import GitCommandError, Repo
+from pkg_resources import parse_version
+
+# Read Towncrier settings
+tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"]
+
+CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
+START_STRING = tc_settings.get(
+    "start_string",
+    "<!-- towncrier release notes start -->\n"
+    if CHANGELOG_FILE.endswith(".md")
+    else ".. towncrier release notes start\n",
+)
+TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})")
+
+
+NAME_REGEX = r".*"
+VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)"
+DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}"
+TITLE_REGEX = (
+    "("
+    + re.escape(
+        TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX")
+    )
+    .replace("NAME_REGEX", NAME_REGEX)
+    .replace("VERSION_REGEX", VERSION_REGEX)
+    .replace("DATE_REGEX", DATE_REGEX)
+    + ")"
+)
+
+
+def get_changelog(repo, branch):
+    return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n"
+
+
+def _tokenize_changes(splits):
+    assert len(splits) % 3 == 0
+    for i in range(len(splits) // 3):
+        title = splits[3 * i]
+        version = parse_version(splits[3 * i + 1])
+        yield [version, title + splits[3 * i + 2]]
+
+
+def split_changelog(changelog):
+    preamble, rest = changelog.split(START_STRING, maxsplit=1)
+    split_rest = re.split(TITLE_REGEX, rest)
+    return preamble + START_STRING + split_rest[0], list(_tokenize_changes(split_rest[1:]))
+
+
+def main():
+    repo = Repo(os.getcwd())
+    remote = repo.remotes[0]
+    branches = [ref for ref in remote.refs if re.match(r"^([0-9]+)\.([0-9]+)$", ref.remote_head)]
+    branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True)
+    branches = [ref.name for ref in branches]
+
+    with open(CHANGELOG_FILE, "r") as f:
+        main_changelog = f.read()
+    preamble, main_changes = split_changelog(main_changelog)
+    old_length = len(main_changes)
+
+    for branch in branches:
+        print(f"Looking at branch {branch}")
+        try:
+            changelog = get_changelog(repo, branch)
+        except GitCommandError:
+            print("No changelog found on this branch.")
+            continue
+        dummy, changes = split_changelog(changelog)
+        new_changes = sorted(main_changes + changes, key=lambda x: x[0], reverse=True)
+        # Now remove duplicates (retain the first one)
+        main_changes = [new_changes[0]]
+        for left, right in itertools.pairwise(new_changes):
+            if left[0] != right[0]:
+                main_changes.append(right)
+
+    new_length = len(main_changes)
+    if old_length < new_length:
+        print(f"{new_length - old_length} new versions have been added.")
+        with open(CHANGELOG_FILE, "w") as fp:
+            fp.write(preamble)
+            for change in main_changes:
+                fp.write(change[1])
+
+        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+
+
+if __name__ == "__main__":
+    main()

.flake8

@@ -6,6 +6,8 @@
 # For more info visit https://github.com/pulp/plugin_template
 [flake8]
 exclude = ./docs/*,*/migrations/*
+per-file-ignores = */__init__.py: F401
+
 ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402
 max-line-length = 100