From ae3a088729f15427b969cf54c5cb4180555661cf Mon Sep 17 00:00:00 2001 From: Ina Panova Date: Fri, 8 Sep 2023 16:42:56 +0200 Subject: [PATCH] Added repo_config json file closes #2902 closes #2903 --- CHANGES/2902.feature | 1 + CHANGES/2903.feature | 1 + docs/workflows/create_sync_publish.rst | 10 +-- .../app/migrations/0054_remove_gpg_fields.py | 29 ++++++++ .../migrations/0055_add_repo_config_field.py | 23 ++++++ pulp_rpm/app/models/repository.py | 34 ++++----- pulp_rpm/app/serializers/repository.py | 71 +++++++++++++++++-- pulp_rpm/app/tasks/publishing.py | 10 ++- pulp_rpm/app/tasks/synchronizing.py | 3 +- pulp_rpm/app/viewsets/repository.py | 19 +++-- .../functional/api/test_consume_content.py | 36 +++++----- pulp_rpm/tests/functional/utils.py | 8 +-- 12 files changed, 178 insertions(+), 67 deletions(-) create mode 100644 CHANGES/2902.feature create mode 100644 CHANGES/2903.feature create mode 100644 pulp_rpm/app/migrations/0054_remove_gpg_fields.py create mode 100644 pulp_rpm/app/migrations/0055_add_repo_config_field.py diff --git a/CHANGES/2902.feature b/CHANGES/2902.feature new file mode 100644 index 0000000000..55abd39441 --- /dev/null +++ b/CHANGES/2902.feature @@ -0,0 +1 @@ +Added new json field repo_config that can be used to configure .repo file diff --git a/CHANGES/2903.feature b/CHANGES/2903.feature new file mode 100644 index 0000000000..55abd39441 --- /dev/null +++ b/CHANGES/2903.feature @@ -0,0 +1 @@ +Added new json field repo_config that can be used to configure .repo file diff --git a/docs/workflows/create_sync_publish.rst b/docs/workflows/create_sync_publish.rst index 5376cf8458..bce3b3d4b1 100644 --- a/docs/workflows/create_sync_publish.rst +++ b/docs/workflows/create_sync_publish.rst @@ -315,14 +315,10 @@ Publication GET response (when task complete): "repository_version": "/pulp/api/v3/repositories/rpm/rpm/a02ace53-d490-458d-8b93-604fbcd23a9c/versions/1/" } -The GPG signature check options are configurable from this REST API endpoint as well. This can be -done via the following options: +The GPG signature check options, like ``gpgcheck`` and ``repo_gpgcheck`` are configurable via the ``repo_config`` option. +This option has a json format and can contain any of the configuration for the ``.repo`` file. -- gpgcheck: perform a GPG signature check on the packages retrieved from this repository. - -- repo_gpgcheck: perform a GPG signature check on the repodata. - -Additionally, an option is provided to let the user decide whether or not to generate sqlite metadata +A separate option is provided to let the user decide whether or not to generate sqlite metadata (defaults to 'false'). Sqlite metadata not commonly used. - sqlite_metadata: generate sqlite metadata in addition to standard XML metadata diff --git a/pulp_rpm/app/migrations/0054_remove_gpg_fields.py b/pulp_rpm/app/migrations/0054_remove_gpg_fields.py new file mode 100644 index 0000000000..43aea4c628 --- /dev/null +++ b/pulp_rpm/app/migrations/0054_remove_gpg_fields.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.4 on 2023-09-08 14:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("rpm", "0053_rpmdistribution_generate_repo_config"), + ] + + operations = [ + migrations.RemoveField( + model_name="rpmpublication", + name="gpgcheck", + ), + migrations.RemoveField( + model_name="rpmpublication", + name="repo_gpgcheck", + ), + migrations.RemoveField( + model_name="rpmrepository", + name="gpgcheck", + ), + migrations.RemoveField( + model_name="rpmrepository", + name="repo_gpgcheck", + ), + ] diff --git a/pulp_rpm/app/migrations/0055_add_repo_config_field.py b/pulp_rpm/app/migrations/0055_add_repo_config_field.py new file mode 100644 index 0000000000..7d37b221b9 --- /dev/null +++ b/pulp_rpm/app/migrations/0055_add_repo_config_field.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.4 on 2023-09-08 14:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("rpm", "0054_remove_gpg_fields"), + ] + + operations = [ + migrations.AddField( + model_name="rpmpublication", + name="repo_config", + field=models.JSONField(default=dict), + ), + migrations.AddField( + model_name="rpmrepository", + name="repo_config", + field=models.JSONField(default=dict), + ), + ] diff --git a/pulp_rpm/app/models/repository.py b/pulp_rpm/app/models/repository.py index 7824ef5e3a..a5bf4a810b 100644 --- a/pulp_rpm/app/models/repository.py +++ b/pulp_rpm/app/models/repository.py @@ -198,12 +198,8 @@ class RpmRepository(Repository, AutoAddObjPermsMixin): The name of a checksum type to use for metadata when generating metadata. package_checksum_type (String): The name of a default checksum type to use for packages when generating metadata. - gpgcheck (Integer): - 1 or 0 corresponding to whether gpgcheck should be enabled in the generated .repo file. - repo_gpgcheck (Integer): - 1 or 0 corresponding to whether repo_gpgcheck should be enabled in the generated - .repo file. sqlite_metadata (Boolean): Whether to generate sqlite metadata files on publish. + repo_config (JSON): repo configuration that will be served by distribution """ TYPE = "rpm" @@ -221,7 +217,6 @@ class RpmRepository(Repository, AutoAddObjPermsMixin): ModulemdObsolete, ] REMOTE_TYPES = [RpmRemote, UlnRemote] - GPGCHECK_CHOICES = [(0, 0), (1, 1)] metadata_signing_service = models.ForeignKey( AsciiArmoredDetachedSigningService, on_delete=models.SET_NULL, null=True @@ -233,9 +228,8 @@ class RpmRepository(Repository, AutoAddObjPermsMixin): autopublish = models.BooleanField(default=False) metadata_checksum_type = models.TextField(null=True, choices=CHECKSUM_CHOICES) package_checksum_type = models.TextField(null=True, choices=CHECKSUM_CHOICES) - gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES) - repo_gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES) sqlite_metadata = models.BooleanField(default=False) + repo_config = models.JSONField(default=dict) def on_new_version(self, version): """ @@ -258,13 +252,13 @@ def on_new_version(self, version): if self.autopublish: tasks.publish( repository_version_pk=version.pk, - gpgcheck_options={"gpgcheck": self.gpgcheck, "repo_gpgcheck": self.repo_gpgcheck}, metadata_signing_service=self.metadata_signing_service, checksum_types={ "metadata": self.metadata_checksum_type, "package": self.package_checksum_type, }, sqlite_metadata=self.sqlite_metadata, + repo_config=self.repo_config, ) @staticmethod @@ -423,14 +417,11 @@ class RpmPublication(Publication, AutoAddObjPermsMixin): Publication for "rpm" content. """ - GPGCHECK_CHOICES = [(0, 0), (1, 1)] - TYPE = "rpm" metadata_checksum_type = models.TextField(choices=CHECKSUM_CHOICES) package_checksum_type = models.TextField(choices=CHECKSUM_CHOICES) - gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES) - repo_gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES) sqlite_metadata = models.BooleanField(default=False) + repo_config = models.JSONField(default=dict) class Meta: default_related_name = "%(app_label)s_%(model_name)s" @@ -474,16 +465,27 @@ def content_handler(self, path): self.base_path, ) ) + repo_config = publication.repo_config + repo_config.pop("name", None) + repo_config.pop("baseurl", None) val = textwrap.dedent( f"""\ [{re.sub(self.INVALID_REPO_ID_CHARS, "", self.name)}] name={self.name} - enabled=1 baseurl={base_url} - gpgcheck={publication.gpgcheck} - repo_gpgcheck={publication.repo_gpgcheck} """ ) + for k, v in repo_config.items(): + val += f"{k}={v}\n" + + if "repo_gpgcheck" not in repo_config: + val += "repo_gpgcheck=0\n" + + if "gpgcheck" not in repo_config: + val += "gpgcheck=0\n" + + if "enabled" not in repo_config: + val += "enabled=1\n" signing_service = repository.metadata_signing_service if signing_service: diff --git a/pulp_rpm/app/serializers/repository.py b/pulp_rpm/app/serializers/repository.py index 2883b6b069..76b37549e5 100644 --- a/pulp_rpm/app/serializers/repository.py +++ b/pulp_rpm/app/serializers/repository.py @@ -1,5 +1,7 @@ from gettext import gettext as _ +import logging + from django.conf import settings from jsonschema import Draft7Validator from rest_framework import serializers @@ -82,20 +84,20 @@ class RpmRepositorySerializer(RepositorySerializer): gpgcheck = serializers.IntegerField( max_value=1, min_value=0, - default=0, required=False, + allow_null=True, help_text=_( - "An option specifying whether a client should perform " + "DEPRECATED: An option specifying whether a client should perform " "a GPG signature check on packages." ), ) repo_gpgcheck = serializers.IntegerField( max_value=1, min_value=0, - default=0, required=False, + allow_null=True, help_text=_( - "An option specifying whether a client should perform " + "DEPRECATED: An option specifying whether a client should perform " "a GPG signature check on the repodata." ), ) @@ -106,6 +108,10 @@ class RpmRepositorySerializer(RepositorySerializer): "DEPRECATED: An option specifying whether Pulp should generate SQLite metadata." ), ) + repo_config = serializers.JSONField( + required=False, + help_text=_("A JSON document describing config.repo file"), + ) def validate(self, data): """Validate data.""" @@ -118,8 +124,45 @@ def validate(self, data): raise serializers.ValidationError({field: _(ALLOWED_CHECKSUM_ERROR_MSG)}) validated_data = super().validate(data) + if ("gpgcheck" or "repo_gpgcheck") in data and "repo_config" in data: + raise serializers.ValidationError( + _( + "Cannot use gpg options and 'repo_config' options simultaneously. " + "The 'gpgcheck' and 'repo_gpgcheck' options aew deprecated, please use " + "'repo_config' only." + ) + ) return validated_data + def create(self, validated_data): + """ + Save the repo and handle gpg options + + Args: + validated_data (dict): A dict of validated data to create the repo + + Returns: + repo: the created repo + """ + gpgcheck = validated_data.pop("gpgcheck", None) + repo_gpgcheck = validated_data.pop("repo_gpgcheck", None) + + gpgcheck_options = {} + if gpgcheck is not None: + gpgcheck_options["gpgcheck"] = gpgcheck + if repo_gpgcheck is not None: + gpgcheck_options["repo_gpgcheck"] = repo_gpgcheck + if gpgcheck_options.keys(): + logging.getLogger("pulp_rpm.deprecation").info( + "Support for gpg options will be removed from a future release of pulp_rpm." + ) + repo_config = ( + gpgcheck_options if gpgcheck_options else validated_data.pop("repo_config", {}) + ) + repo = super().create(validated_data) + repo.repo_config = repo_config + return repo + class Meta: fields = RepositorySerializer.Meta.fields + ( "autopublish", @@ -130,6 +173,7 @@ class Meta: "gpgcheck", "repo_gpgcheck", "sqlite_metadata", + "repo_config", ) model = RpmRepository @@ -223,8 +267,9 @@ class RpmPublicationSerializer(PublicationSerializer): max_value=1, min_value=0, required=False, + allow_null=True, help_text=_( - "An option specifying whether a client should perform " + "DEPRECATED: An option specifying whether a client should perform " "a GPG signature check on packages." ), ) @@ -232,8 +277,9 @@ class RpmPublicationSerializer(PublicationSerializer): max_value=1, min_value=0, required=False, + allow_null=True, help_text=_( - "An option specifying whether a client should perform " + "DEPRECATED: An option specifying whether a client should perform " "a GPG signature check on the repodata." ), ) @@ -244,6 +290,10 @@ class RpmPublicationSerializer(PublicationSerializer): "DEPRECATED: An option specifying whether Pulp should generate SQLite metadata." ), ) + repo_config = serializers.JSONField( + required=False, + help_text=_("A JSON document describing config.repo file"), + ) def validate(self, data): """Validate data.""" @@ -256,6 +306,14 @@ def validate(self, data): ): raise serializers.ValidationError(_(ALLOWED_CHECKSUM_ERROR_MSG)) validated_data = super().validate(data) + if ("gpgcheck" or "repo_gpgcheck") in data and "repo_config" in data: + raise serializers.ValidationError( + _( + "Cannot use gpg options and 'repo_config' options simultaneously. " + "The 'gpgcheck' and 'repo_gpgcheck' options are deprecated, please use " + "'repo_config' only." + ) + ) return validated_data class Meta: @@ -265,6 +323,7 @@ class Meta: "gpgcheck", "repo_gpgcheck", "sqlite_metadata", + "repo_config", ) model = RpmPublication diff --git a/pulp_rpm/app/tasks/publishing.py b/pulp_rpm/app/tasks/publishing.py index 34c1c00d2c..17d8829d76 100644 --- a/pulp_rpm/app/tasks/publishing.py +++ b/pulp_rpm/app/tasks/publishing.py @@ -320,21 +320,21 @@ def cr_checksum_type_from_string(checksum_type): def publish( repository_version_pk, - gpgcheck_options=None, metadata_signing_service=None, checksum_types=None, sqlite_metadata=False, + repo_config=None, ): """ Create a Publication based on a RepositoryVersion. Args: repository_version_pk (str): Create a publication from this repository version. - gpgcheck_options (dict): GPG signature check options. metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService): A reference to an associated signing service. checksum_types (dict): Checksum types for metadata and packages. sqlite_metadata (bool): Whether to generate metadata files in sqlite format. + repo_config (JSON): repo config that will be served by distribution """ repository_version = RepositoryVersion.objects.get(pk=repository_version_pk) @@ -361,13 +361,11 @@ def publish( checksum_types.get("package") or publication.metadata_checksum_type ) - if gpgcheck_options is not None: - publication.gpgcheck = gpgcheck_options.get("gpgcheck") - publication.repo_gpgcheck = gpgcheck_options.get("repo_gpgcheck") - if sqlite_metadata: publication.sqlite_metadata = True + publication.repo_config = repo_config + publication_data = PublicationData(publication) publication_data.populate() diff --git a/pulp_rpm/app/tasks/synchronizing.py b/pulp_rpm/app/tasks/synchronizing.py index 3375f9c691..30b04824b8 100644 --- a/pulp_rpm/app/tasks/synchronizing.py +++ b/pulp_rpm/app/tasks/synchronizing.py @@ -158,8 +158,7 @@ def add_metadata_to_publication(publication, version, prefix=""): publication.package_checksum_type = CHECKSUM_TYPES.UNKNOWN publication.metadata_checksum_type = CHECKSUM_TYPES.UNKNOWN - publication.gpgcheck = 0 - publication.repo_gpgcheck = has_repomd_signature + publication.repo_config = {"repo_gpgcheck": has_repomd_signature, "gpgcheck": 0} publication.sqlite_metadata = has_sqlite for relative_path, metadata_file_path in repo_metadata_files.items(): diff --git a/pulp_rpm/app/viewsets/repository.py b/pulp_rpm/app/viewsets/repository.py index 3f555dfc2c..1d31ebd617 100644 --- a/pulp_rpm/app/viewsets/repository.py +++ b/pulp_rpm/app/viewsets/repository.py @@ -547,10 +547,19 @@ def create(self, request): metadata=metadata_checksum_type, package=package_checksum_type, ) - gpgcheck_options = dict( - gpgcheck=serializer.validated_data.get("gpgcheck", repository.gpgcheck), - repo_gpgcheck=serializer.validated_data.get("repo_gpgcheck", repository.repo_gpgcheck), - ) + gpgcheck = serializer.validated_data.get("gpgcheck") + repo_gpgcheck = serializer.validated_data.get("repo_gpgcheck") + gpgcheck_options = {} + if gpgcheck is not None: + gpgcheck_options["gpgcheck"] = gpgcheck + if repo_gpgcheck is not None: + gpgcheck_options["repo_gpgcheck"] = repo_gpgcheck + if gpgcheck_options.keys(): + logging.getLogger("pulp_rpm.deprecation").info( + "Support for gpg options will be removed from a future release of pulp_rpm." + ) + repo_config = serializer.validated_data.get("repo_config", repository.repo_config) + repo_config = gpgcheck_options if gpgcheck_options else repo_config sqlite_metadata = serializer.validated_data.get( "sqlite_metadata", repository.sqlite_metadata ) @@ -572,7 +581,7 @@ def create(self, request): "repository_version_pk": repository_version.pk, "metadata_signing_service": signing_service_pk, "checksum_types": checksum_types, - "gpgcheck_options": gpgcheck_options, + "repo_config": repo_config, "sqlite_metadata": sqlite_metadata, }, ) diff --git a/pulp_rpm/tests/functional/api/test_consume_content.py b/pulp_rpm/tests/functional/api/test_consume_content.py index 104e797c37..26137f07e8 100644 --- a/pulp_rpm/tests/functional/api/test_consume_content.py +++ b/pulp_rpm/tests/functional/api/test_consume_content.py @@ -60,8 +60,7 @@ def create_distribution( rpm_distribution_factory, ): def _create_distribution( - gpgcheck=None, - repo_gpgcheck=None, + repo_config=None, has_signing_service=False, generate_repo_config=False, repo_body=None, @@ -76,10 +75,8 @@ def _create_distribution( repo, _ = init_and_sync(repository=repo, url=url, policy=policy, sync_policy=sync_policy) pub_body = {"repository": repo.pulp_href} - if gpgcheck is not None: - pub_body["gpgcheck"] = gpgcheck - if repo_gpgcheck is not None: - pub_body["repo_gpgcheck"] = repo_gpgcheck + if repo_config is not None: + pub_body["repo_config"] = repo_config publication = rpm_publication_factory(**pub_body) return rpm_distribution_factory( @@ -160,7 +157,7 @@ def test_publish_signed_repo_metadata( if rpm_metadata_signing_service is None: pytest.skip("Need a signing service for this test") - distribution = create_distribution(gpgcheck=0, repo_gpgcheck=0, has_signing_service=True) + distribution = create_distribution(repo_config={}, has_signing_service=True) dnf_config_add_repo(distribution, has_signing_service=True) rpm_name = "walrus" @@ -175,10 +172,9 @@ def test_publish_signed_repo_metadata( assert rpm_name == rpm[0] -# Test all possible combinations of gpgcheck options made to a publication. +# Test all possible combinations of options made to a publication. test_options = { - "gpgcheck": [0, 1], - "repo_gpgcheck": [0, 1], + "repo_config": [{}, {"assumeyes": True, "gpgcheck": 1}], "has_signing_service": [True, False], "generate_repo_config": [True, False], } @@ -186,12 +182,9 @@ def test_publish_signed_repo_metadata( @pytest.mark.parallel -@pytest.mark.parametrize( - "gpgcheck,repo_gpgcheck,has_signing_service,generate_repo_config", func_params -) +@pytest.mark.parametrize("repo_config,has_signing_service,generate_repo_config", func_params) def test_config_dot_repo( - gpgcheck, - repo_gpgcheck, + repo_config, has_signing_service, generate_repo_config, rpm_metadata_signing_service, @@ -205,8 +198,7 @@ def test_config_dot_repo( pytest.skip("Generation of config.repo file was disabled") distribution = create_distribution( - gpgcheck=gpgcheck, - repo_gpgcheck=repo_gpgcheck, + repo_config=repo_config, has_signing_service=has_signing_service, generate_repo_config=generate_repo_config, ) @@ -214,12 +206,16 @@ def test_config_dot_repo( assert f"[{distribution.name}]\n" in content assert f"baseurl={distribution.base_url}\n" in content - assert f"gpgcheck={gpgcheck}\n" in content - assert f"repo_gpgcheck={repo_gpgcheck}\n" in content + assert "gpgcheck=0\n" in content + assert "repo_gpgcheck=0\n" in content if has_signing_service: assert f"gpgkey={distribution.base_url}repodata/repomd.xml.key" in content + if repo_config: + assert "assumeyes=True\n" in content + assert "gpgcheck=1\n" in content + @pytest.mark.parallel def test_repomd_headers( @@ -228,7 +224,7 @@ def test_repomd_headers( ): """Test if repomd.xml is returned with Cache-control: no-cache header.""" distribution = create_distribution( - gpgcheck=1, repo_gpgcheck=1, has_signing_service=True, generate_repo_config=True + repo_config={}, has_signing_service=True, generate_repo_config=True ) assert ( http_get_headers(f"{distribution.base_url}repodata/repomd.xml").get("Cache-control", "") diff --git a/pulp_rpm/tests/functional/utils.py b/pulp_rpm/tests/functional/utils.py index 1a649158e7..99152e196a 100644 --- a/pulp_rpm/tests/functional/utils.py +++ b/pulp_rpm/tests/functional/utils.py @@ -97,16 +97,14 @@ def rpm_copy(cfg, config, recursive=False): return client.post(RPM_COPY_PATH, data) -def publish(cfg, repo, version_href=None, gpgcheck=0, repo_gpgcheck=0): +def publish(cfg, repo, version_href=None, repo_config=None): """Publish a repository. :param pulp_smash.config.PulpSmashConfig cfg: Information about the Pulp host. :param repo: A dict of information about the repository. :param version_href: A href for the repo version to be published. - :param gpgcheck: An option specifying whether to perform a GPG signature check on packages. - :param repo_gpgcheck: An option specifying whether to perform a GPG signature check on - the repodata. + :param repo_config: An option specifying config for .repo file :returns: A publication. A dict of information about the just created publication. """ @@ -115,7 +113,7 @@ def publish(cfg, repo, version_href=None, gpgcheck=0, repo_gpgcheck=0): else: body = {"repository": repo["pulp_href"]} - body.update({"gpgcheck": gpgcheck, "repo_gpgcheck": repo_gpgcheck}) + body.update({"repo_config": repo_config}) client = api.Client(cfg, api.json_handler) call_report = client.post(RPM_PUBLICATION_PATH, body)