Remove sqlite metadata support

closes #2457

CHANGES/2457.removal

@@ -0,0 +1 @@
+Removed the ability to generate sqlite metadata during repository publish.

coverage.md

@@ -41,7 +41,6 @@ This file contains list of features and their test coverage.
 | As a user, I can sign repository metadata using a signing service and publish such repo | PART |  |
 | As a user, I can have a config.repo file generated for any distribution at runtime | YES |  |
 | As a user I can set/update repo_gpgcheck and gpg_check options | YES | |
-| As a user I can set/update option to publish sqlite metadata | YES | |
 | **Upload** |  |  |
 | As a user, I can upload rpm packages, advisories and modulemd[-defaults] content types and optionally add them to repository | NO |  |
 | As a user, I can upload rpm packages, advisories and modulemd[-defaults] type of content in chunks | PART |  |

docs/workflows/create_sync_publish.rst

@@ -318,11 +318,6 @@ Publication GET response (when task complete):
 The GPG signature check options, like ``gpgcheck`` and ``repo_gpgcheck`` are configurable via the ``repo_config`` option.
 This option has a json format and can contain any of the configuration for the ``.repo`` file.
 
-A separate option is provided to let the user decide whether or not to generate sqlite metadata
-(defaults to 'false'). Sqlite metadata not commonly used.
-
-- sqlite_metadata: generate sqlite metadata in addition to standard XML metadata
-
 We encourage users to take a look at the `pulp_rpm API documentation <../restapi.html#operation/publications_rpm_rpm_create>`_
 to see the default values for these options.
 

pulp_rpm/app/migrations/0056_remove_rpmpublication_sqlite_metadata_and_more.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.7 on 2023-11-22 18:48
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("rpm", "0055_add_repo_config_field"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="rpmpublication",
+            name="sqlite_metadata",
+        ),
+        migrations.RemoveField(
+            model_name="rpmrepository",
+            name="sqlite_metadata",
+        ),
+    ]

pulp_rpm/app/models/repository.py

@@ -198,7 +198,6 @@ class RpmRepository(Repository, AutoAddObjPermsMixin):
             The name of a checksum type to use for metadata when generating metadata.
         package_checksum_type (String):
             The name of a default checksum type to use for packages when generating metadata.
-        sqlite_metadata (Boolean): Whether to generate sqlite metadata files on publish.
         repo_config (JSON): repo configuration that will be served by distribution
     """
 
@@ -228,7 +227,6 @@ class RpmRepository(Repository, AutoAddObjPermsMixin):
     autopublish = models.BooleanField(default=False)
     metadata_checksum_type = models.TextField(null=True, choices=CHECKSUM_CHOICES)
     package_checksum_type = models.TextField(null=True, choices=CHECKSUM_CHOICES)
-    sqlite_metadata = models.BooleanField(default=False)
     repo_config = models.JSONField(default=dict)
 
     def on_new_version(self, version):
@@ -243,12 +241,6 @@ def on_new_version(self, version):
         # avoid circular import issues
         from pulp_rpm.app import tasks
 
-        if self.sqlite_metadata:
-            getLogger("pulp_rpm.deprecation").info(
-                "Support for sqlite metadata generation will be removed from a future release "
-                "of pulp_rpm. See https://tinyurl.com/sqlite-removal for more details"
-            )
-
         if self.autopublish:
             tasks.publish(
                 repository_version_pk=version.pk,
@@ -257,7 +249,6 @@ def on_new_version(self, version):
                     "metadata": self.metadata_checksum_type,
                     "package": self.package_checksum_type,
                 },
-                sqlite_metadata=self.sqlite_metadata,
                 repo_config=self.repo_config,
             )
 
@@ -424,7 +415,6 @@ class RpmPublication(Publication, AutoAddObjPermsMixin):
     TYPE = "rpm"
     metadata_checksum_type = models.TextField(choices=CHECKSUM_CHOICES)
     package_checksum_type = models.TextField(choices=CHECKSUM_CHOICES)
-    sqlite_metadata = models.BooleanField(default=False)
     repo_config = models.JSONField(default=dict)
 
     class Meta:

pulp_rpm/app/serializers/repository.py

@@ -107,6 +107,7 @@ class RpmRepositorySerializer(RepositorySerializer):
         help_text=_(
             "DEPRECATED: An option specifying whether Pulp should generate SQLite metadata."
         ),
+        read_only=True,
     )
     repo_config = serializers.JSONField(
         required=False,
@@ -291,6 +292,7 @@ class RpmPublicationSerializer(PublicationSerializer):
         help_text=_(
             "DEPRECATED: An option specifying whether Pulp should generate SQLite metadata."
         ),
+        read_only=True,
     )
     repo_config = serializers.JSONField(
         required=False,

pulp_rpm/app/tasks/publishing.py

@@ -322,7 +322,6 @@ def publish(
     repository_version_pk,
     metadata_signing_service=None,
     checksum_types=None,
-    sqlite_metadata=False,
     repo_config=None,
 ):
     """
@@ -333,7 +332,6 @@ def publish(
         metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService):
             A reference to an associated signing service.
         checksum_types (dict): Checksum types for metadata and packages.
-        sqlite_metadata (bool): Whether to generate metadata files in sqlite format.
         repo_config (JSON): repo config that will be served by distribution
 
     """
@@ -361,9 +359,6 @@ def publish(
                 checksum_types.get("package") or publication.metadata_checksum_type
             )
 
-            if sqlite_metadata:
-                publication.sqlite_metadata = True
-
             publication.repo_config = repo_config
 
             publication_data = PublicationData(publication)
@@ -458,14 +453,6 @@ def generate_repo_metadata(
     oth_xml = cr.OtherXmlFile(oth_xml_path, compressiontype=cr.GZ)
     upd_xml = None
 
-    if publication.sqlite_metadata:
-        pri_db_path = os.path.join(cwd, "primary.sqlite")
-        fil_db_path = os.path.join(cwd, "filelists.sqlite")
-        oth_db_path = os.path.join(cwd, "other.sqlite")
-        pri_db = cr.PrimarySqlite(pri_db_path)
-        fil_db = cr.FilelistsSqlite(fil_db_path)
-        oth_db = cr.OtherSqlite(oth_db_path)
-
     # We want to support publishing with a different checksum type than the one built-in to the
     # package itself, so we need to get the correct checksums somehow if there is an override.
     # We must also take into consideration that if the package has not been downloaded the only
@@ -573,10 +560,6 @@ def generate_repo_metadata(
         pri_xml.add_pkg(pkg)
         fil_xml.add_pkg(pkg)
         oth_xml.add_pkg(pkg)
-        if publication.sqlite_metadata:
-            pri_db.add_pkg(pkg)
-            fil_db.add_pkg(pkg)
-            oth_db.add_pkg(pkg)
 
     # Process update records
     update_records = UpdateRecord.objects.filter(pk__in=content).order_by("id", "digest")
@@ -650,21 +633,11 @@ def generate_repo_metadata(
     if not content.exists():
         repomd.revision = "0"
 
-    if publication.sqlite_metadata:
-        repomdrecords = [
-            ("primary", pri_xml_path, pri_db),
-            ("filelists", fil_xml_path, fil_db),
-            ("other", oth_xml_path, oth_db),
-            ("primary_db", pri_db_path, None),
-            ("filelists_db", fil_db_path, None),
-            ("other_db", oth_db_path, None),
-        ]
-    else:
-        repomdrecords = [
-            ("primary", pri_xml_path, None),
-            ("filelists", fil_xml_path, None),
-            ("other", oth_xml_path, None),
-        ]
+    repomdrecords = [
+        ("primary", pri_xml_path, None),
+        ("filelists", fil_xml_path, None),
+        ("other", oth_xml_path, None),
+    ]
 
     if upd_xml:
         repomdrecords.append(("updateinfo", upd_xml_path, None))
@@ -677,27 +650,18 @@ def generate_repo_metadata(
 
     repomdrecords.extend(extra_repomdrecords)
 
-    sqlite_files = ("primary_db", "filelists_db", "other_db")
-
     for name, path, db_to_update in repomdrecords:
         record = cr.RepomdRecord(name, path)
         checksum_type = cr_checksum_type_from_string(
             get_checksum_type(name, checksum_types, default=publication.metadata_checksum_type)
         )
-        if name in sqlite_files:
-            record_bz = record.compress_and_fill(checksum_type, cr.BZ2)
-            record_bz.type = name
-            record_bz.rename_file()
-            path = record_bz.location_href.split("/")[-1]
-            repomd.set_record(record_bz)
-        else:
-            record.fill(checksum_type)
-            if db_to_update:
-                db_to_update.dbinfo_update(record.checksum)
-                db_to_update.close()
-            record.rename_file()
-            path = record.location_href.split("/")[-1]
-            repomd.set_record(record)
+        record.fill(checksum_type)
+        if db_to_update:
+            db_to_update.dbinfo_update(record.checksum)
+            db_to_update.close()
+        record.rename_file()
+        path = record.location_href.split("/")[-1]
+        repomd.set_record(record)
 
         if sub_folder:
             path = os.path.join(sub_folder, path)

pulp_rpm/app/tasks/synchronizing.py

@@ -154,12 +154,10 @@ def add_metadata_to_publication(publication, version, prefix=""):
     repo_metadata_files = metadata_files_for_mirroring[str(version.repository.pk)]
 
     has_repomd_signature = "repodata/repomd.xml.asc" in repo_metadata_files.keys()
-    has_sqlite = any([".sqlite" in href for href in repo_metadata_files.keys()])
 
     publication.package_checksum_type = CHECKSUM_TYPES.UNKNOWN
     publication.metadata_checksum_type = CHECKSUM_TYPES.UNKNOWN
     publication.repo_config = {"repo_gpgcheck": has_repomd_signature, "gpgcheck": 0}
-    publication.sqlite_metadata = has_sqlite
 
     for relative_path, metadata_file_path in repo_metadata_files.items():
         with open(metadata_file_path, "rb") as metadata_fd:

pulp_rpm/app/viewsets/repository.py

@@ -562,14 +562,6 @@ def create(self, request):
             )
         repo_config = serializer.validated_data.get("repo_config", repository.repo_config)
         repo_config = gpgcheck_options if gpgcheck_options else repo_config
-        sqlite_metadata = serializer.validated_data.get(
-            "sqlite_metadata", repository.sqlite_metadata
-        )
-        if sqlite_metadata:
-            logging.getLogger("pulp_rpm.deprecation").info(
-                "Support for sqlite metadata generation will be removed from a future release "
-                "of pulp_rpm. See https://tinyurl.com/sqlite-removal for more details"
-            )
 
         if repository.metadata_signing_service:
             signing_service_pk = repository.metadata_signing_service.pk
@@ -584,7 +576,6 @@ def create(self, request):
                 "metadata_signing_service": signing_service_pk,
                 "checksum_types": checksum_types,
                 "repo_config": repo_config,
-                "sqlite_metadata": sqlite_metadata,
             },
         )
         return OperationPostponedResponse(result, request)

pulp_rpm/tests/functional/api/test_auto_publish.py

@@ -12,7 +12,7 @@
 def setup_autopublish(rpm_repository_factory, rpm_rpmremote_factory, rpm_distribution_factory):
     """Create remote, repo, publish settings, and distribution."""
     remote = rpm_rpmremote_factory()
-    repo = rpm_repository_factory(autopublish=True, sqlite_metadata=True)
+    repo = rpm_repository_factory(autopublish=True, metadata_checksum_type="sha512")
     distribution = rpm_distribution_factory(repository=repo.pulp_href)
 
     return repo, remote, distribution
@@ -37,7 +37,7 @@ def test_01_sync(setup_autopublish, rpm_repository_api, rpm_publication_api, mon
 
     # Check that the publish settings were used
     publication = publications.results[0]
-    assert publication.sqlite_metadata is True
+    assert publication.metadata_checksum_type == "sha512"
 
     # Sync the repository again. Since there should be no new repository version, there
     # should be no new publications or distributions either.
@@ -70,4 +70,4 @@ def test_02_modify(
 
     # Check that the publish settings were used
     publication = publications.results[0]
-    assert publication.sqlite_metadata is True
+    assert publication.metadata_checksum_type == "sha512"

pulp_rpm/tests/functional/api/test_publish.py

@@ -35,55 +35,6 @@
 
 
 class TestPublishWithUnsignedRepoSyncedOnDemand:
-    @pytest.mark.parallel
-    @pytest.mark.parametrize("with_sqlite", [True, False], ids=["with_sqlite", "without_sqlite"])
-    def test_sqlite_metadata(
-        self,
-        with_sqlite,
-        rpm_unsigned_repo_on_demand,
-        rpm_publication_api,
-        gen_object_with_cleanup,
-        rpm_distribution_api,
-        monitor_task,
-    ):
-        """Publish repository and validate the updateinfo.
-
-        This Test does the following:
-
-        1. Create a rpm repo and a remote.
-        2. Sync the repo with the remote.
-        3. Publish with and without sqlite metadata and distribute the repo.
-        4. Verify that the sqlite metadata files are/not present when expected.
-        """
-        publish_data = RpmRpmPublication(
-            repository=rpm_unsigned_repo_on_demand.pulp_href, sqlite_metadata=with_sqlite
-        )
-        publish_response = rpm_publication_api.create(publish_data)
-        created_resources = monitor_task(publish_response.task).created_resources
-        publication_href = created_resources[0]
-
-        body = gen_distribution(publication=publication_href)
-        distribution = gen_object_with_cleanup(rpm_distribution_api, body)
-
-        repomd = ElementTree.fromstring(
-            requests.get(os.path.join(distribution.base_url, "repodata/repomd.xml")).text
-        )
-
-        data_xpath = "{{{}}}data".format(RPM_NAMESPACES["metadata/repo"])
-        data_elems = [elem for elem in repomd.findall(data_xpath)]
-
-        sqlite_files = [elem for elem in data_elems if elem.get("type").endswith("_db")]
-
-        if with_sqlite:
-            assert 3 == len(sqlite_files)
-
-            for db_elem in sqlite_files:
-                location_xpath = "{{{}}}location".format(RPM_NAMESPACES["metadata/repo"])
-                db_href = db_elem.find(location_xpath).get("href")
-                assert requests.get(os.path.join(distribution.base_url, db_href)).status_code == 200
-        else:
-            assert 0 == len(sqlite_files)
-
     @pytest.mark.parallel
     def test_publish_with_unsupported_checksum_type(
         self, rpm_unsigned_repo_on_demand, rpm_publication_api