Remove the ability to use checksums weaker than sha256 during publish

closes #2488

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-250-gf4315cf
+2021.08.26-251-g45d399d

.github/workflows/scripts/install.sh

@@ -68,7 +68,7 @@ VARSYAML
 
 cat >> vars/main.yaml << VARSYAML
 pulp_env: {}
-pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "orphan_protection_time": 0}
+pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "orphan_protection_time": 0}
 pulp_scheme: https
 
 pulp_container_tag: "latest"

.github/workflows/scripts/release.py

@@ -57,11 +57,7 @@ def create_release_commits(repo, release_version, plugin_path):
     # Second commit: release version
     os.system("bump2version release --allow-dirty")
 
-    git.add(f"{plugin_path}/pulp_rpm/*")
-    git.add(f"{plugin_path}/docs/conf.py")
-    git.add(f"{plugin_path}/setup.py")
-    git.add(f"{plugin_path}/requirements.txt")
-    git.add(f"{plugin_path}/.bumpversion.cfg")
+    git.add(f"{plugin_path}")
     git.commit("-m", f"Release {release_version}\nGH Issues: {issues}\n\n[noissue]")
     sha = repo.head.object.hexsha
     short_sha = git.rev_parse(sha, short=7)
@@ -76,11 +72,7 @@ def create_release_commits(repo, release_version, plugin_path):
         if not new_dev_version:
             raise RuntimeError("Could not detect new dev version ... aborting.")
 
-    git.add(f"{plugin_path}/pulp_rpm/*")
-    git.add(f"{plugin_path}/docs/conf.py")
-    git.add(f"{plugin_path}/setup.py")
-    git.add(f"{plugin_path}/requirements.txt")
-    git.add(f"{plugin_path}/.bumpversion.cfg")
+    git.add(f"{plugin_path}")
     git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]")
     print(f"Release commit == {short_sha}")
     print(f"All changes were committed on branch: release_{release_version}")
@@ -144,6 +136,7 @@ def main():
         "release_version",
         type=str,
         help="The version string for the release.",
+        nargs="?",
     )
 
     args = parser.parse_args()
@@ -160,7 +153,7 @@ def main():
     repo = Repo(plugin_path)
 
     release_commit = None
-    if release_version != release_version_arg:
+    if release_version_arg and release_version != release_version_arg:
         # Look for a commit with the requested release version
         for commit in repo.iter_commits():
             if f"Release {release_version_arg}\n" in commit.message:

CHANGES/2488.removal

@@ -0,0 +1 @@
+Removed support for publishing repos with a checksum type of md5, sha1, or sha224

pulp_rpm/app/constants.py

@@ -13,11 +13,6 @@
 
 # The same as above, but in a format that choice fields can use
 CHECKSUM_CHOICES = (
-    (CHECKSUM_TYPES.UNKNOWN, CHECKSUM_TYPES.UNKNOWN),
-    (CHECKSUM_TYPES.MD5, CHECKSUM_TYPES.MD5),
-    (CHECKSUM_TYPES.SHA, CHECKSUM_TYPES.SHA1),
-    (CHECKSUM_TYPES.SHA1, CHECKSUM_TYPES.SHA1),
-    (CHECKSUM_TYPES.SHA224, CHECKSUM_TYPES.SHA224),
     (CHECKSUM_TYPES.SHA256, CHECKSUM_TYPES.SHA256),
     (CHECKSUM_TYPES.SHA384, CHECKSUM_TYPES.SHA384),
     (CHECKSUM_TYPES.SHA512, CHECKSUM_TYPES.SHA512),

pulp_rpm/app/migrations/0056_rpmpublication_checksum_type_and_more.py

@@ -6,7 +6,11 @@
 
 def set_publication_checksum(apps, schema_editor):
     RpmPublication = apps.get_model("rpm", "RpmPublication")
-    RpmPublication.objects.update(checksum_type=F("metadata_checksum_type"))
+    RpmPublication.objects.update(checksum_type="sha256")
+
+def set_repository_checksum(apps, schema_editor):
+    RpmRepository = apps.get_model("rpm", "RpmRepository")
+    RpmRepository.objects.update(checksum_type="sha256")
 
 
 class Migration(migrations.Migration):
@@ -32,4 +36,10 @@ class Migration(migrations.Migration):
             name='checksum_type',
             field=models.TextField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], null=True),
         ),
+        migrations.RunPython(set_repository_checksum),
+        migrations.AlterField(
+            model_name='rpmrepository',
+            name='checksum_type',
+            field=models.TextField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')]),
+        ),
     ]

pulp_rpm/app/migrations/0057_alter_package_checksum_type_and_more.py

@@ -0,0 +1,53 @@
+# Generated by Django 4.2.5 on 2023-11-07 05:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rpm', '0056_rpmpublication_checksum_type_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='package',
+            name='checksum_type',
+            field=models.TextField(choices=[('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')]),
+        ),
+        migrations.AlterField(
+            model_name='repometadatafile',
+            name='checksum_type',
+            field=models.TextField(choices=[('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')]),
+        ),
+        migrations.AlterField(
+            model_name='rpmpublication',
+            name='checksum_type',
+            field=models.TextField(choices=[('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')]),
+        ),
+        migrations.AlterField(
+            model_name='rpmpublication',
+            name='metadata_checksum_type',
+            field=models.TextField(choices=[('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')]),
+        ),
+        migrations.AlterField(
+            model_name='rpmpublication',
+            name='package_checksum_type',
+            field=models.TextField(choices=[('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')]),
+        ),
+        migrations.AlterField(
+            model_name='rpmrepository',
+            name='checksum_type',
+            field=models.TextField(choices=[('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], null=True),
+        ),
+        migrations.AlterField(
+            model_name='rpmrepository',
+            name='metadata_checksum_type',
+            field=models.TextField(choices=[('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], null=True),
+        ),
+        migrations.AlterField(
+            model_name='rpmrepository',
+            name='package_checksum_type',
+            field=models.TextField(choices=[('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], null=True),
+        ),
+    ]

pulp_rpm/app/models/repository.py

@@ -261,6 +261,7 @@ def on_new_version(self, version):
                 repository_version_pk=version.pk,
                 metadata_signing_service=self.metadata_signing_service,
                 checksum_types={
+                    "general": self.checksum_type,
                     "metadata": self.metadata_checksum_type,
                     "package": self.package_checksum_type,
                     "general": self.checksum_type,

pulp_rpm/app/tasks/publishing.py

@@ -304,7 +304,8 @@ def get_checksum_type(name, checksum_types, default=CHECKSUM_TYPES.SHA256):
     """
     original = checksum_types.get("original")
     metadata = checksum_types.get("metadata")
-    checksum_type = metadata if metadata else original.get(name, default)
+    general = checksum_types.get("general", metadata)
+    checksum_type = general if general else original.get(name, default)
     # "sha" -> "SHA" -> "CHECKSUM_TYPES.SHA" -> "sha1"
     normalized_checksum_type = getattr(CHECKSUM_TYPES, checksum_type.upper())
     return normalized_checksum_type

pulp_rpm/tests/functional/api/test_download_content.py

@@ -40,11 +40,6 @@ def test_all(
     2. Select a random content unit in the distribution. Download that
        content unit from Pulp, and verify that the content unit has the
        same checksum when fetched directly from Pulp-Fixtures.
-
-    This test targets the following issues:
-
-    * `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
-    * `Pulp Smash #872 <https://github.com/pulp/pulp-smash/issues/872>`_
     """
     # Sync a Repository
     repo = rpm_unsigned_repo_immediate

pulp_rpm/tests/functional/api/test_download_policies.py

@@ -8,16 +8,6 @@
 from pulpcore.client.pulp_rpm import RpmRpmPublication
 
 
-"""Sync a repository with different download policies.
-
-This test targets the following issue:
-
-`Pulp #4126 <https://pulp.plan.io/issues/4126>`_
-`Pulp #4213 <https://pulp.plan.io/issues/4213>`_
-`Pulp #4418 <https://pulp.plan.io/issues/4418>`_
-"""
-
-
 @pytest.mark.parametrize("download_policy", DOWNLOAD_POLICIES)
 def test_download_policies(
     download_policy,

pulp_rpm/tests/functional/api/test_publish.py

@@ -92,18 +92,18 @@ def test_publish_with_unsupported_checksum_type(
         Sync and try to publish an RPM repository.
 
         - Sync repository with on_demand policy
-        - Try to publish with 'md5' checksum type
-        - Publish should fail because 'md5' is not allowed
+        - Try to publish with 'sha384' checksum type
+        - Publish should fail because 'sha384' is not allowed
 
-        This test require disallowed 'MD5' checksum type from ALLOWED_CONTENT_CHECKSUMS settings.
+        This test require disallowed 'sha384' checksum type from ALLOWED_CONTENT_CHECKSUMS settings.
         """
-        if "md5" in settings.ALLOWED_CONTENT_CHECKSUMS:
+        if "sha384" in settings.ALLOWED_CONTENT_CHECKSUMS:
             pytest.skip(
-                reason="Cannot verify the expected hasher error if the 'MD5' checksum is allowed."
+                reason="Cannot check for the expected error if the 'sha384' checksum is allowed."
             )
 
         publish_data = RpmRpmPublication(
-            repository=rpm_unsigned_repo_on_demand.pulp_href, package_checksum_type="md5"
+            repository=rpm_unsigned_repo_on_demand.pulp_href, package_checksum_type="sha384"
         )
         with pytest.raises(ApiException) as ctx:
             rpm_publication_api.create(publish_data)
@@ -632,14 +632,14 @@ def test_immediate_unspecified_checksum_types(get_checksum_types):
 def test_on_demand_specified_package_checksum_type(get_checksum_types, delete_orphans_pre):
     """Sync and publish an RPM repository and verify the checksum types."""
     repomd_checksum_types, primary_checksum_types = get_checksum_types(
-        package_checksum_type="sha384", policy="on_demand"
+        package_checksum_type="sha512", policy="on_demand"
     )
 
     for repomd_type, repomd_checksum_type in repomd_checksum_types.items():
         assert repomd_checksum_type == "sha256"
 
     for package, package_checksum_type in primary_checksum_types.items():
-        # since none of the packages in question have sha384 checksums, the
+        # since none of the packages in question have sha512 checksums, the
         # checksums they do have will be used instead. In this case, sha256.
         assert package_checksum_type == "sha256"
 
@@ -648,11 +648,11 @@ def test_on_demand_specified_package_checksum_type(get_checksum_types, delete_or
 def test_on_demand_specified_metadata_checksum_type(get_checksum_types):
     """Sync and publish an RPM repository and verify the checksum types."""
     repomd_checksum_types, primary_checksum_types = get_checksum_types(
-        metadata_checksum_type="sha384", policy="on_demand"
+        metadata_checksum_type="sha512", policy="on_demand"
     )
 
     for repomd_type, repomd_checksum_type in repomd_checksum_types.items():
-        assert repomd_checksum_type == "sha384"
+        assert repomd_checksum_type == "sha512"
 
     for package, package_checksum_type in primary_checksum_types.items():
         assert package_checksum_type == "sha256"
@@ -663,14 +663,14 @@ def test_on_demand_specified_metadata_and_package_checksum_type(
 ):
     """Sync and publish an RPM repository and verify the checksum types."""
     repomd_checksum_types, primary_checksum_types = get_checksum_types(
-        package_checksum_type="sha224", metadata_checksum_type="sha224", policy="on_demand"
+        package_checksum_type="sha512", metadata_checksum_type="sha512", policy="on_demand"
     )
 
     for repomd_type, repomd_checksum_type in repomd_checksum_types.items():
-        assert repomd_checksum_type == "sha224"
+        assert repomd_checksum_type == "sha512"
 
     for package, package_checksum_type in primary_checksum_types.items():
-        # since none of the packages in question have sha224 checksums, the
+        # since none of the packages in question have sha512 checksums, the
         # checksums they do have will be used instead. In this case, sha256.
         assert package_checksum_type == "sha256"
 
@@ -679,25 +679,25 @@ def test_on_demand_specified_metadata_and_package_checksum_type(
 def test_immediate_specified_package_checksum_type(get_checksum_types):
     """Sync and publish an RPM repository and verify the checksum types."""
     repomd_checksum_types, primary_checksum_types = get_checksum_types(
-        package_checksum_type="sha384", policy="immediate"
+        package_checksum_type="sha512", policy="immediate"
     )
 
     for repomd_type, repomd_checksum_type in repomd_checksum_types.items():
         assert repomd_checksum_type == "sha256"
 
     for package, package_checksum_type in primary_checksum_types.items():
-        assert package_checksum_type == "sha384"
+        assert package_checksum_type == "sha512"
 
 
 @pytest.mark.parallel
 def test_immediate_specified_metadata_checksum_type(get_checksum_types):
     """Sync and publish an RPM repository and verify the checksum types."""
     repomd_checksum_types, primary_checksum_types = get_checksum_types(
-        metadata_checksum_type="sha384", policy="immediate"
+        metadata_checksum_type="sha512", policy="immediate"
     )
 
     for repomd_type, repomd_checksum_type in repomd_checksum_types.items():
-        assert repomd_checksum_type == "sha384"
+        assert repomd_checksum_type == "sha512"
 
     for package, package_checksum_type in primary_checksum_types.items():
         assert package_checksum_type == "sha256"
@@ -707,14 +707,14 @@ def test_immediate_specified_metadata_checksum_type(get_checksum_types):
 def test_immediate_specified_metadata_and_package_checksum_type(get_checksum_types):
     """Sync and publish an RPM repository and verify the checksum types."""
     repomd_checksum_types, primary_checksum_types = get_checksum_types(
-        package_checksum_type="sha224", metadata_checksum_type="sha224", policy="immediate"
+        package_checksum_type="sha512", metadata_checksum_type="sha512", policy="immediate"
     )
 
     for repomd_type, repomd_checksum_type in repomd_checksum_types.items():
-        assert repomd_checksum_type == "sha224"
+        assert repomd_checksum_type == "sha512"
 
     for package, package_checksum_type in primary_checksum_types.items():
-        assert package_checksum_type == "sha224"
+        assert package_checksum_type == "sha512"
 
 
 @pytest.mark.parallel

pulp_rpm/tests/functional/api/test_pulp_to_pulp.py

@@ -32,8 +32,7 @@ def test_pulp_pulp_sync(
     # Create a publication.
     publish_data = RpmRpmPublication(
         repository=repo.pulp_href,
-        metadata_checksum_type="sha384",
-        package_checksum_type="sha224",
+        checksum_type="sha512",
     )
     publication = gen_object_with_cleanup(rpm_publication_api, publish_data)
 

template_config.yml

@@ -60,7 +60,6 @@ pulp_settings:
   - sha1
   - sha224
   - sha256
-  - sha384
   - sha512
   allowed_export_paths:
   - /tmp