From 2d773510ddc5fd54cecfede453d822fef6570c55 Mon Sep 17 00:00:00 2001 From: Ina Panova Date: Fri, 8 Sep 2023 16:42:56 +0200 Subject: [PATCH] Added repo_config json file closes #2902 closes #2903 --- CHANGES/2902.feature | 1 + CHANGES/2903.feature | 1 + docs/workflows/create_sync_publish.rst | 10 ++-- ...remove_rpmpublication_gpgcheck_and_more.py | 39 ++++++++++++++ pulp_rpm/app/models/repository.py | 34 ++++++------ pulp_rpm/app/serializers/repository.py | 54 +++++-------------- pulp_rpm/app/tasks/publishing.py | 10 ++-- pulp_rpm/app/tasks/synchronizing.py | 3 +- pulp_rpm/app/viewsets/repository.py | 7 +-- .../functional/api/test_consume_content.py | 36 ++++++------- pulp_rpm/tests/functional/utils.py | 8 ++- 11 files changed, 100 insertions(+), 103 deletions(-) create mode 100644 CHANGES/2902.feature create mode 100644 CHANGES/2903.feature create mode 100644 pulp_rpm/app/migrations/0054_remove_rpmpublication_gpgcheck_and_more.py diff --git a/CHANGES/2902.feature b/CHANGES/2902.feature new file mode 100644 index 0000000000..55abd39441 --- /dev/null +++ b/CHANGES/2902.feature @@ -0,0 +1 @@ +Added new json field repo_config that can be used to configure .repo file diff --git a/CHANGES/2903.feature b/CHANGES/2903.feature new file mode 100644 index 0000000000..55abd39441 --- /dev/null +++ b/CHANGES/2903.feature @@ -0,0 +1 @@ +Added new json field repo_config that can be used to configure .repo file diff --git a/docs/workflows/create_sync_publish.rst b/docs/workflows/create_sync_publish.rst index 5376cf8458..bce3b3d4b1 100644 --- a/docs/workflows/create_sync_publish.rst +++ b/docs/workflows/create_sync_publish.rst @@ -315,14 +315,10 @@ Publication GET response (when task complete): "repository_version": "/pulp/api/v3/repositories/rpm/rpm/a02ace53-d490-458d-8b93-604fbcd23a9c/versions/1/" } -The GPG signature check options are configurable from this REST API endpoint as well. This can be -done via the following options: +The GPG signature check options, like ``gpgcheck`` and ``repo_gpgcheck`` are configurable via the ``repo_config`` option. +This option has a json format and can contain any of the configuration for the ``.repo`` file. -- gpgcheck: perform a GPG signature check on the packages retrieved from this repository. - -- repo_gpgcheck: perform a GPG signature check on the repodata. - -Additionally, an option is provided to let the user decide whether or not to generate sqlite metadata +A separate option is provided to let the user decide whether or not to generate sqlite metadata (defaults to 'false'). Sqlite metadata not commonly used. - sqlite_metadata: generate sqlite metadata in addition to standard XML metadata diff --git a/pulp_rpm/app/migrations/0054_remove_rpmpublication_gpgcheck_and_more.py b/pulp_rpm/app/migrations/0054_remove_rpmpublication_gpgcheck_and_more.py new file mode 100644 index 0000000000..fcdafb74ac --- /dev/null +++ b/pulp_rpm/app/migrations/0054_remove_rpmpublication_gpgcheck_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.4 on 2023-09-08 14:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("rpm", "0053_rpmdistribution_generate_repo_config"), + ] + + operations = [ + migrations.RemoveField( + model_name="rpmpublication", + name="gpgcheck", + ), + migrations.RemoveField( + model_name="rpmpublication", + name="repo_gpgcheck", + ), + migrations.RemoveField( + model_name="rpmrepository", + name="gpgcheck", + ), + migrations.RemoveField( + model_name="rpmrepository", + name="repo_gpgcheck", + ), + migrations.AddField( + model_name="rpmpublication", + name="repo_config", + field=models.JSONField(default=dict), + ), + migrations.AddField( + model_name="rpmrepository", + name="repo_config", + field=models.JSONField(default=dict), + ), + ] diff --git a/pulp_rpm/app/models/repository.py b/pulp_rpm/app/models/repository.py index 7824ef5e3a..a5bf4a810b 100644 --- a/pulp_rpm/app/models/repository.py +++ b/pulp_rpm/app/models/repository.py @@ -198,12 +198,8 @@ class RpmRepository(Repository, AutoAddObjPermsMixin): The name of a checksum type to use for metadata when generating metadata. package_checksum_type (String): The name of a default checksum type to use for packages when generating metadata. - gpgcheck (Integer): - 1 or 0 corresponding to whether gpgcheck should be enabled in the generated .repo file. - repo_gpgcheck (Integer): - 1 or 0 corresponding to whether repo_gpgcheck should be enabled in the generated - .repo file. sqlite_metadata (Boolean): Whether to generate sqlite metadata files on publish. + repo_config (JSON): repo configuration that will be served by distribution """ TYPE = "rpm" @@ -221,7 +217,6 @@ class RpmRepository(Repository, AutoAddObjPermsMixin): ModulemdObsolete, ] REMOTE_TYPES = [RpmRemote, UlnRemote] - GPGCHECK_CHOICES = [(0, 0), (1, 1)] metadata_signing_service = models.ForeignKey( AsciiArmoredDetachedSigningService, on_delete=models.SET_NULL, null=True @@ -233,9 +228,8 @@ class RpmRepository(Repository, AutoAddObjPermsMixin): autopublish = models.BooleanField(default=False) metadata_checksum_type = models.TextField(null=True, choices=CHECKSUM_CHOICES) package_checksum_type = models.TextField(null=True, choices=CHECKSUM_CHOICES) - gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES) - repo_gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES) sqlite_metadata = models.BooleanField(default=False) + repo_config = models.JSONField(default=dict) def on_new_version(self, version): """ @@ -258,13 +252,13 @@ def on_new_version(self, version): if self.autopublish: tasks.publish( repository_version_pk=version.pk, - gpgcheck_options={"gpgcheck": self.gpgcheck, "repo_gpgcheck": self.repo_gpgcheck}, metadata_signing_service=self.metadata_signing_service, checksum_types={ "metadata": self.metadata_checksum_type, "package": self.package_checksum_type, }, sqlite_metadata=self.sqlite_metadata, + repo_config=self.repo_config, ) @staticmethod @@ -423,14 +417,11 @@ class RpmPublication(Publication, AutoAddObjPermsMixin): Publication for "rpm" content. """ - GPGCHECK_CHOICES = [(0, 0), (1, 1)] - TYPE = "rpm" metadata_checksum_type = models.TextField(choices=CHECKSUM_CHOICES) package_checksum_type = models.TextField(choices=CHECKSUM_CHOICES) - gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES) - repo_gpgcheck = models.IntegerField(default=0, choices=GPGCHECK_CHOICES) sqlite_metadata = models.BooleanField(default=False) + repo_config = models.JSONField(default=dict) class Meta: default_related_name = "%(app_label)s_%(model_name)s" @@ -474,16 +465,27 @@ def content_handler(self, path): self.base_path, ) ) + repo_config = publication.repo_config + repo_config.pop("name", None) + repo_config.pop("baseurl", None) val = textwrap.dedent( f"""\ [{re.sub(self.INVALID_REPO_ID_CHARS, "", self.name)}] name={self.name} - enabled=1 baseurl={base_url} - gpgcheck={publication.gpgcheck} - repo_gpgcheck={publication.repo_gpgcheck} """ ) + for k, v in repo_config.items(): + val += f"{k}={v}\n" + + if "repo_gpgcheck" not in repo_config: + val += "repo_gpgcheck=0\n" + + if "gpgcheck" not in repo_config: + val += "gpgcheck=0\n" + + if "enabled" not in repo_config: + val += "enabled=1\n" signing_service = repository.metadata_signing_service if signing_service: diff --git a/pulp_rpm/app/serializers/repository.py b/pulp_rpm/app/serializers/repository.py index 2883b6b069..b199bffa8c 100644 --- a/pulp_rpm/app/serializers/repository.py +++ b/pulp_rpm/app/serializers/repository.py @@ -79,26 +79,7 @@ class RpmRepositorySerializer(RepositorySerializer): required=False, allow_null=True, ) - gpgcheck = serializers.IntegerField( - max_value=1, - min_value=0, - default=0, - required=False, - help_text=_( - "An option specifying whether a client should perform " - "a GPG signature check on packages." - ), - ) - repo_gpgcheck = serializers.IntegerField( - max_value=1, - min_value=0, - default=0, - required=False, - help_text=_( - "An option specifying whether a client should perform " - "a GPG signature check on the repodata." - ), - ) + sqlite_metadata = serializers.BooleanField( default=False, required=False, @@ -106,6 +87,10 @@ class RpmRepositorySerializer(RepositorySerializer): "DEPRECATED: An option specifying whether Pulp should generate SQLite metadata." ), ) + repo_config = serializers.JSONField( + required=False, + help_text=_("A JSON document describing config.repo file"), + ) def validate(self, data): """Validate data.""" @@ -127,9 +112,8 @@ class Meta: "retain_package_versions", "metadata_checksum_type", "package_checksum_type", - "gpgcheck", - "repo_gpgcheck", "sqlite_metadata", + "repo_config", ) model = RpmRepository @@ -219,24 +203,6 @@ class RpmPublicationSerializer(PublicationSerializer): choices=CHECKSUM_CHOICES, required=False, ) - gpgcheck = serializers.IntegerField( - max_value=1, - min_value=0, - required=False, - help_text=_( - "An option specifying whether a client should perform " - "a GPG signature check on packages." - ), - ) - repo_gpgcheck = serializers.IntegerField( - max_value=1, - min_value=0, - required=False, - help_text=_( - "An option specifying whether a client should perform " - "a GPG signature check on the repodata." - ), - ) sqlite_metadata = serializers.BooleanField( default=False, required=False, @@ -245,6 +211,11 @@ class RpmPublicationSerializer(PublicationSerializer): ), ) + repo_config = serializers.JSONField( + required=False, + help_text=_("A JSON document describing config.repo file"), + ) + def validate(self, data): """Validate data.""" if ( @@ -262,9 +233,8 @@ class Meta: fields = PublicationSerializer.Meta.fields + ( "metadata_checksum_type", "package_checksum_type", - "gpgcheck", - "repo_gpgcheck", "sqlite_metadata", + "repo_config", ) model = RpmPublication diff --git a/pulp_rpm/app/tasks/publishing.py b/pulp_rpm/app/tasks/publishing.py index 34c1c00d2c..17d8829d76 100644 --- a/pulp_rpm/app/tasks/publishing.py +++ b/pulp_rpm/app/tasks/publishing.py @@ -320,21 +320,21 @@ def cr_checksum_type_from_string(checksum_type): def publish( repository_version_pk, - gpgcheck_options=None, metadata_signing_service=None, checksum_types=None, sqlite_metadata=False, + repo_config=None, ): """ Create a Publication based on a RepositoryVersion. Args: repository_version_pk (str): Create a publication from this repository version. - gpgcheck_options (dict): GPG signature check options. metadata_signing_service (pulpcore.app.models.AsciiArmoredDetachedSigningService): A reference to an associated signing service. checksum_types (dict): Checksum types for metadata and packages. sqlite_metadata (bool): Whether to generate metadata files in sqlite format. + repo_config (JSON): repo config that will be served by distribution """ repository_version = RepositoryVersion.objects.get(pk=repository_version_pk) @@ -361,13 +361,11 @@ def publish( checksum_types.get("package") or publication.metadata_checksum_type ) - if gpgcheck_options is not None: - publication.gpgcheck = gpgcheck_options.get("gpgcheck") - publication.repo_gpgcheck = gpgcheck_options.get("repo_gpgcheck") - if sqlite_metadata: publication.sqlite_metadata = True + publication.repo_config = repo_config + publication_data = PublicationData(publication) publication_data.populate() diff --git a/pulp_rpm/app/tasks/synchronizing.py b/pulp_rpm/app/tasks/synchronizing.py index 3375f9c691..30b04824b8 100644 --- a/pulp_rpm/app/tasks/synchronizing.py +++ b/pulp_rpm/app/tasks/synchronizing.py @@ -158,8 +158,7 @@ def add_metadata_to_publication(publication, version, prefix=""): publication.package_checksum_type = CHECKSUM_TYPES.UNKNOWN publication.metadata_checksum_type = CHECKSUM_TYPES.UNKNOWN - publication.gpgcheck = 0 - publication.repo_gpgcheck = has_repomd_signature + publication.repo_config = {"repo_gpgcheck": has_repomd_signature, "gpgcheck": 0} publication.sqlite_metadata = has_sqlite for relative_path, metadata_file_path in repo_metadata_files.items(): diff --git a/pulp_rpm/app/viewsets/repository.py b/pulp_rpm/app/viewsets/repository.py index 3f555dfc2c..f8cfeed9b0 100644 --- a/pulp_rpm/app/viewsets/repository.py +++ b/pulp_rpm/app/viewsets/repository.py @@ -547,10 +547,7 @@ def create(self, request): metadata=metadata_checksum_type, package=package_checksum_type, ) - gpgcheck_options = dict( - gpgcheck=serializer.validated_data.get("gpgcheck", repository.gpgcheck), - repo_gpgcheck=serializer.validated_data.get("repo_gpgcheck", repository.repo_gpgcheck), - ) + repo_config = serializer.validated_data.get("repo_config", repository.repo_config) sqlite_metadata = serializer.validated_data.get( "sqlite_metadata", repository.sqlite_metadata ) @@ -572,7 +569,7 @@ def create(self, request): "repository_version_pk": repository_version.pk, "metadata_signing_service": signing_service_pk, "checksum_types": checksum_types, - "gpgcheck_options": gpgcheck_options, + "repo_config": repo_config, "sqlite_metadata": sqlite_metadata, }, ) diff --git a/pulp_rpm/tests/functional/api/test_consume_content.py b/pulp_rpm/tests/functional/api/test_consume_content.py index 104e797c37..26137f07e8 100644 --- a/pulp_rpm/tests/functional/api/test_consume_content.py +++ b/pulp_rpm/tests/functional/api/test_consume_content.py @@ -60,8 +60,7 @@ def create_distribution( rpm_distribution_factory, ): def _create_distribution( - gpgcheck=None, - repo_gpgcheck=None, + repo_config=None, has_signing_service=False, generate_repo_config=False, repo_body=None, @@ -76,10 +75,8 @@ def _create_distribution( repo, _ = init_and_sync(repository=repo, url=url, policy=policy, sync_policy=sync_policy) pub_body = {"repository": repo.pulp_href} - if gpgcheck is not None: - pub_body["gpgcheck"] = gpgcheck - if repo_gpgcheck is not None: - pub_body["repo_gpgcheck"] = repo_gpgcheck + if repo_config is not None: + pub_body["repo_config"] = repo_config publication = rpm_publication_factory(**pub_body) return rpm_distribution_factory( @@ -160,7 +157,7 @@ def test_publish_signed_repo_metadata( if rpm_metadata_signing_service is None: pytest.skip("Need a signing service for this test") - distribution = create_distribution(gpgcheck=0, repo_gpgcheck=0, has_signing_service=True) + distribution = create_distribution(repo_config={}, has_signing_service=True) dnf_config_add_repo(distribution, has_signing_service=True) rpm_name = "walrus" @@ -175,10 +172,9 @@ def test_publish_signed_repo_metadata( assert rpm_name == rpm[0] -# Test all possible combinations of gpgcheck options made to a publication. +# Test all possible combinations of options made to a publication. test_options = { - "gpgcheck": [0, 1], - "repo_gpgcheck": [0, 1], + "repo_config": [{}, {"assumeyes": True, "gpgcheck": 1}], "has_signing_service": [True, False], "generate_repo_config": [True, False], } @@ -186,12 +182,9 @@ def test_publish_signed_repo_metadata( @pytest.mark.parallel -@pytest.mark.parametrize( - "gpgcheck,repo_gpgcheck,has_signing_service,generate_repo_config", func_params -) +@pytest.mark.parametrize("repo_config,has_signing_service,generate_repo_config", func_params) def test_config_dot_repo( - gpgcheck, - repo_gpgcheck, + repo_config, has_signing_service, generate_repo_config, rpm_metadata_signing_service, @@ -205,8 +198,7 @@ def test_config_dot_repo( pytest.skip("Generation of config.repo file was disabled") distribution = create_distribution( - gpgcheck=gpgcheck, - repo_gpgcheck=repo_gpgcheck, + repo_config=repo_config, has_signing_service=has_signing_service, generate_repo_config=generate_repo_config, ) @@ -214,12 +206,16 @@ def test_config_dot_repo( assert f"[{distribution.name}]\n" in content assert f"baseurl={distribution.base_url}\n" in content - assert f"gpgcheck={gpgcheck}\n" in content - assert f"repo_gpgcheck={repo_gpgcheck}\n" in content + assert "gpgcheck=0\n" in content + assert "repo_gpgcheck=0\n" in content if has_signing_service: assert f"gpgkey={distribution.base_url}repodata/repomd.xml.key" in content + if repo_config: + assert "assumeyes=True\n" in content + assert "gpgcheck=1\n" in content + @pytest.mark.parallel def test_repomd_headers( @@ -228,7 +224,7 @@ def test_repomd_headers( ): """Test if repomd.xml is returned with Cache-control: no-cache header.""" distribution = create_distribution( - gpgcheck=1, repo_gpgcheck=1, has_signing_service=True, generate_repo_config=True + repo_config={}, has_signing_service=True, generate_repo_config=True ) assert ( http_get_headers(f"{distribution.base_url}repodata/repomd.xml").get("Cache-control", "") diff --git a/pulp_rpm/tests/functional/utils.py b/pulp_rpm/tests/functional/utils.py index 1a649158e7..99152e196a 100644 --- a/pulp_rpm/tests/functional/utils.py +++ b/pulp_rpm/tests/functional/utils.py @@ -97,16 +97,14 @@ def rpm_copy(cfg, config, recursive=False): return client.post(RPM_COPY_PATH, data) -def publish(cfg, repo, version_href=None, gpgcheck=0, repo_gpgcheck=0): +def publish(cfg, repo, version_href=None, repo_config=None): """Publish a repository. :param pulp_smash.config.PulpSmashConfig cfg: Information about the Pulp host. :param repo: A dict of information about the repository. :param version_href: A href for the repo version to be published. - :param gpgcheck: An option specifying whether to perform a GPG signature check on packages. - :param repo_gpgcheck: An option specifying whether to perform a GPG signature check on - the repodata. + :param repo_config: An option specifying config for .repo file :returns: A publication. A dict of information about the just created publication. """ @@ -115,7 +113,7 @@ def publish(cfg, repo, version_href=None, gpgcheck=0, repo_gpgcheck=0): else: body = {"repository": repo["pulp_href"]} - body.update({"gpgcheck": gpgcheck, "repo_gpgcheck": repo_gpgcheck}) + body.update({"repo_config": repo_config}) client = api.Client(cfg, api.json_handler) call_report = client.post(RPM_PUBLICATION_PATH, body)