From c6a717695f5518b751a06d56978a0e0c40842c9b Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Thu, 1 Jul 2021 10:19:20 -0400 Subject: [PATCH 01/79] Properly specifies the package name that needs to be installed [noissue] --- .github/workflows/scripts/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 532bc5318e..2f8e9fabf4 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -38,7 +38,7 @@ else PULP_CERTGUARD=git+https://github.com/pulp/pulp-certguard.git@master fi if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - PLUGIN_NAME=./dist/pulpcore-* + PLUGIN_NAME=./pulpcore/dist/pulpcore-$PLUGIN_VERSION-py3-none-any.whl else PLUGIN_NAME=./pulpcore fi From f91ddcf8d32e88058b694fd13c3e57e2e1105730 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 1 Jul 2021 14:24:16 +0000 Subject: [PATCH 02/79] Building changelog for 3.14.0 [noissue] --- CHANGES.rst | 170 ++++++++++++++++++++++++++++ CHANGES/6837.doc | 1 - CHANGES/8501.feature | 1 - CHANGES/8637.bugfix | 1 - CHANGES/8658.feature | 4 - CHANGES/8670.bugfix | 1 - CHANGES/8708.bugfix | 3 - CHANGES/8774.doc | 1 - CHANGES/8800.doc | 1 - CHANGES/8801.doc | 1 - CHANGES/8803.doc | 1 - CHANGES/8805.feature | 1 - CHANGES/8821.misc | 1 - CHANGES/8827.misc | 1 - CHANGES/8867.bugfix | 1 - CHANGES/8868.bugfix | 9 -- CHANGES/8869.bugfix | 2 - CHANGES/8876.deprecation | 2 - CHANGES/8881.feature | 1 - CHANGES/8883.bugfix | 1 - CHANGES/8891.feature | 1 - CHANGES/8897.feature | 1 - CHANGES/8926.bugfix | 2 - CHANGES/8930.feature | 1 - CHANGES/8946.doc | 1 - CHANGES/8947.doc | 1 - CHANGES/8948.feature | 6 - CHANGES/8949.doc | 1 - CHANGES/8975.misc | 1 - CHANGES/8980.bugfix | 2 - CHANGES/plugin_api/8480.deprecation | 2 - CHANGES/plugin_api/8480.feature | 2 - CHANGES/plugin_api/8487.feature | 1 - CHANGES/plugin_api/8501.deprecation | 2 - CHANGES/plugin_api/8501.removal | 2 - CHANGES/plugin_api/8729.removal | 3 - CHANGES/plugin_api/8823.feature | 5 - CHANGES/plugin_api/8827.removal | 1 - CHANGES/plugin_api/8844.feature | 1 - CHANGES/plugin_api/8868.bugfix | 9 -- CHANGES/plugin_api/8881.deprecation | 1 - CHANGES/plugin_api/8881.feature | 1 - CHANGES/plugin_api/8897.feature | 1 - 43 files changed, 170 insertions(+), 82 deletions(-) delete mode 100644 CHANGES/6837.doc delete mode 100644 CHANGES/8501.feature delete mode 100644 CHANGES/8637.bugfix delete mode 100644 CHANGES/8658.feature delete mode 100644 CHANGES/8670.bugfix delete mode 100644 CHANGES/8708.bugfix delete mode 100644 CHANGES/8774.doc delete mode 100644 CHANGES/8800.doc delete mode 100644 CHANGES/8801.doc delete mode 100644 CHANGES/8803.doc delete mode 100644 CHANGES/8805.feature delete mode 100644 CHANGES/8821.misc delete mode 100644 CHANGES/8827.misc delete mode 100644 CHANGES/8867.bugfix delete mode 100644 CHANGES/8868.bugfix delete mode 100644 CHANGES/8869.bugfix delete mode 100644 CHANGES/8876.deprecation delete mode 100644 CHANGES/8881.feature delete mode 100644 CHANGES/8883.bugfix delete mode 100644 CHANGES/8891.feature delete mode 100644 CHANGES/8897.feature delete mode 100644 CHANGES/8926.bugfix delete mode 100644 CHANGES/8930.feature delete mode 100644 CHANGES/8946.doc delete mode 100644 CHANGES/8947.doc delete mode 100644 CHANGES/8948.feature delete mode 100644 CHANGES/8949.doc delete mode 100644 CHANGES/8975.misc delete mode 100644 CHANGES/8980.bugfix delete mode 100644 CHANGES/plugin_api/8480.deprecation delete mode 100644 CHANGES/plugin_api/8480.feature delete mode 100644 CHANGES/plugin_api/8487.feature delete mode 100644 CHANGES/plugin_api/8501.deprecation delete mode 100644 CHANGES/plugin_api/8501.removal delete mode 100644 CHANGES/plugin_api/8729.removal delete mode 100644 CHANGES/plugin_api/8823.feature delete mode 100644 CHANGES/plugin_api/8827.removal delete mode 100644 CHANGES/plugin_api/8844.feature delete mode 100644 CHANGES/plugin_api/8868.bugfix delete mode 100644 CHANGES/plugin_api/8881.deprecation delete mode 100644 CHANGES/plugin_api/8881.feature delete mode 100644 CHANGES/plugin_api/8897.feature diff --git a/CHANGES.rst b/CHANGES.rst index 7232a78d77..6cdae1352f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,176 @@ Changelog .. towncrier release notes start +3.14.0 (2021-07-01)REST API +-------- + +Features +~~~~~~~~ + +- Introduce new worker style. (tech-preview) + `#8501 `_ +- Added new endpoint ``/pulp/api/v3/orphans/cleanup/``. When called with ``POST`` and no parameters + it is equivalent to calling ``DELETE /pulp/api/v3/orphans/``. Additionally the optional parameter + ``content_hrefs`` can be specified and must contain a list of content hrefs. When ``content_hrefs`` + is specified, only those content units will be considered to be removed by orphan cleanup. + `#8658 `_ +- Content app responses are now smartly cached in Redis. + `#8805 `_ +- Downloads from remote sources will now be retried on more kinds of errors, such as HTTP 500 or socket errors. + `#8881 `_ +- Add a correlation id filter to the task list endpoint. + `#8891 `_ +- Where before ``download_concurrency`` would previously be set to a default value upon creation, it will now be set NULL (but a default value will still be used). + `#8897 `_ +- Added graceful shutdown to pulpcore workers. + `#8930 `_ +- Activate the new task worker type by default. + + .. warning:: + + If you intend to stick with the old tasking system, you should configure the + ``USE_NEW_WORKER_TYPE`` setting to false before upgrading. + `#8948 `_ + + +Bugfixes +~~~~~~~~ + +- Fixed race condition where a task could clean up reserved resources shared with another task. + `#8637 `_ +- Altered redirect URL escaping, preventing invalidation of signed URLs for artifacts using cloud storage. + `#8670 `_ +- Add an update row lock on in task dispatching for ``ReservedResource`` to prevent a race where an + object was deleted that was supposed to be reused. This prevents a condition where tasks ended up in + waiting state forever. + `#8708 `_ +- Retry downloads on ``ClientConnectorSSLError``, which appears to be spuriously returned by some CDNs. + `#8867 `_ +- Fixed OpenAPI schema tag generation for resources that are nested more than 2 levels. + + This change is most evident in client libraries generated from the OpenAPI schema. + + Prior to this change, the API client for a resource located at + `/api/v3/pulp/exporters/core/pulp//exports/` was named `ExportersCoreExportsApi`. + + After this change, the API client for a resource located at + `/api/v3/pulp/exporters/core/pulp//exports/` is named `ExportersPulpExportsApi`. + `#8868 `_ +- Fixed request schema for ``/pulp/api/v3/repair/``, which did identify any arguments. This also fixes + the bindings. + `#8869 `_ +- Update default access policies in the database if they were unmodified by the administrator. + `#8883 `_ +- Pinning to psycopg2 < 2.9 as psycopg 2.9 doesn't work with django 2.2. More info at + https://github.com/django/django/commit/837ffcfa681d0f65f444d881ee3d69aec23770be. + `#8926 `_ +- Fixed bug where artifacts and content were not always saved in Pulp with each + on_demand request serviced by content app. + `#8980 `_ + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Fixed a number of link-problems in the installation/ section of docs. + `#6837 `_ +- Added a troubleshooting section to the docs explaining how to find stuck tasks. + `#8774 `_ +- Moved existing basic auth docs to a new top-level section named Authentication. + `#8800 `_ +- Moved ``Webserver Authentication`` docs under the top-level ``Authentication`` section. + `#8801 `_ +- Provide instructions to use Keycloak authenication using Python Social Aauth + `#8803 `_ +- Updated the docs.pulpproject.org to provide some immediate direction for better user orientation. + `#8946 `_ +- Separated hardware and Filesystem information from the Architecture section and added them to the Installation section. + `#8947 `_ +- Added sub-headings and simplified language of Pulp concept section. + `#8949 `_ + + +Deprecations +~~~~~~~~~~~~ + +- Deprecated the ``DELETE /pulp/api/v3/orphans/`` call. Instead use the + ``POST /pulp/api/v3/orphans/cleanup/`` call. + `#8876 `_ + + +Misc +~~~~ + +- `#8821 `_, `#8827 `_, `#8975 `_ + + +Plugin API +---------- + +Features +~~~~~~~~ + +- Added the ``pulpcore.plugin.viewsets.DistributionFilter``. This should be used instead of + ``pulpcore.plugin.viewsets.NewDistributionFilter``. + `#8480 `_ +- Added ``user_hidden`` field to ``Repository`` to hide repositories from users. + `#8487 `_ +- Added a ``timestamp_of_interest`` field to Content and Artifacts. This field can be updated by + calling a new method ``touch()`` on Artifacts and Content. Plugin writers should call this method + whenever they deal with Content or Artifacts. For example, this includes places where Content is + uploaded or added to Repository Versions. This will prevent Content and Artifacts from being cleaned + up when orphan cleanup becomes a non-blocking task in pulpcore 3.15. + `#8823 `_ +- Exposed ``AsyncUpdateMixin`` through ``pulpcore.plugin.viewsets``. + `#8844 `_ +- Added a field ``DEFAULT_MAX_RETRIES`` to the ``Remote`` base class - plugin writers can override the default number of retries attempted when file downloads failed for each type of remote. The default value is 3. + `#8881 `_ +- Added a field ``DEFAULT_DOWNLOAD_CONCURRENCY`` to the Remote base class - plugin writers can override the number of concurrent downloads for each type of remote. The default value is 10. + `#8897 `_ + + +Bugfixes +~~~~~~~~ + +- Fixed OpenAPI schema tag generation for resources that are nested more than 2 levels. + + This change is most evident in client libraries generated from the OpenAPI schema. + + Prior to this change, the API client for a resource located at + `/api/v3/pulp/exporters/core/pulp//exports/` was named `ExportersCoreExportsApi`. + + After this change, the API client for a resource located at + `/api/v3/pulp/exporters/core/pulp//exports/` is named `ExportersPulpExportsApi`. + `#8868 `_ + + +Removals +~~~~~~~~ + +- The usage of non-JSON serializable types of ``args`` and ``kwargs`` to tasks is no longer supported. + ``uuid.UUID`` objects however will silently be converted to ``str``. + `#8501 `_ +- Removed the ``versions_containing_content`` method from the + `pulpcore.plugin.models.RepositoryVersion`` object. Instead use + ``RepositoryVersion.objects.with_content()``. + `#8729 `_ +- Removed `pulpcore.plugin.stages.ContentUnassociation` from the plugin API. + `#8827 `_ + + +Deprecations +~~~~~~~~~~~~ + +- The ``pulpcore.plugin.viewsets.NewDistributionFilter`` is deprecated and will be removed from a + future release. Instead use ``pulpcore.plugin.viewsets.DistributionFilter``. + `#8480 `_ +- Deprecate the use of the `reserved_resources_record__resource` in favor of `reserved_resources_record__contains`. + Tentative removal release is pulpcore==3.15. + `#8501 `_ +- Plugin writers who create custom downloaders by subclassing ``HttpDownloader`` no longer need to wrap the ``_run()`` method with a ``backoff`` decorator. Consequntly the ``http_giveup`` handler the sake of the ``backoff`` decorator is no longer needed and has been deprecated. It is likely to be removed in pulpcore 3.15. + `#8881 `_ + + 3.13.0 (2021-05-25) =================== REST API diff --git a/CHANGES/6837.doc b/CHANGES/6837.doc deleted file mode 100644 index a035220daa..0000000000 --- a/CHANGES/6837.doc +++ /dev/null @@ -1 +0,0 @@ -Fixed a number of link-problems in the installation/ section of docs. diff --git a/CHANGES/8501.feature b/CHANGES/8501.feature deleted file mode 100644 index c14735d2b3..0000000000 --- a/CHANGES/8501.feature +++ /dev/null @@ -1 +0,0 @@ -Introduce new worker style. (tech-preview) diff --git a/CHANGES/8637.bugfix b/CHANGES/8637.bugfix deleted file mode 100644 index f7aad12a57..0000000000 --- a/CHANGES/8637.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed race condition where a task could clean up reserved resources shared with another task. diff --git a/CHANGES/8658.feature b/CHANGES/8658.feature deleted file mode 100644 index 81b48d50e2..0000000000 --- a/CHANGES/8658.feature +++ /dev/null @@ -1,4 +0,0 @@ -Added new endpoint ``/pulp/api/v3/orphans/cleanup/``. When called with ``POST`` and no parameters -it is equivalent to calling ``DELETE /pulp/api/v3/orphans/``. Additionally the optional parameter -``content_hrefs`` can be specified and must contain a list of content hrefs. When ``content_hrefs`` -is specified, only those content units will be considered to be removed by orphan cleanup. diff --git a/CHANGES/8670.bugfix b/CHANGES/8670.bugfix deleted file mode 100644 index 7faea3dc19..0000000000 --- a/CHANGES/8670.bugfix +++ /dev/null @@ -1 +0,0 @@ -Altered redirect URL escaping, preventing invalidation of signed URLs for artifacts using cloud storage. diff --git a/CHANGES/8708.bugfix b/CHANGES/8708.bugfix deleted file mode 100644 index 8aaf81a120..0000000000 --- a/CHANGES/8708.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Add an update row lock on in task dispatching for ``ReservedResource`` to prevent a race where an -object was deleted that was supposed to be reused. This prevents a condition where tasks ended up in -waiting state forever. diff --git a/CHANGES/8774.doc b/CHANGES/8774.doc deleted file mode 100644 index d50ffbaeae..0000000000 --- a/CHANGES/8774.doc +++ /dev/null @@ -1 +0,0 @@ -Added a troubleshooting section to the docs explaining how to find stuck tasks. diff --git a/CHANGES/8800.doc b/CHANGES/8800.doc deleted file mode 100644 index b83f6ba017..0000000000 --- a/CHANGES/8800.doc +++ /dev/null @@ -1 +0,0 @@ -Moved existing basic auth docs to a new top-level section named Authentication. diff --git a/CHANGES/8801.doc b/CHANGES/8801.doc deleted file mode 100644 index bd9f02f97e..0000000000 --- a/CHANGES/8801.doc +++ /dev/null @@ -1 +0,0 @@ -Moved ``Webserver Authentication`` docs under the top-level ``Authentication`` section. diff --git a/CHANGES/8803.doc b/CHANGES/8803.doc deleted file mode 100644 index 8b5d02caaa..0000000000 --- a/CHANGES/8803.doc +++ /dev/null @@ -1 +0,0 @@ -Provide instructions to use Keycloak authenication using Python Social Aauth diff --git a/CHANGES/8805.feature b/CHANGES/8805.feature deleted file mode 100644 index 3d28bffe54..0000000000 --- a/CHANGES/8805.feature +++ /dev/null @@ -1 +0,0 @@ -Content app responses are now smartly cached in Redis. diff --git a/CHANGES/8821.misc b/CHANGES/8821.misc deleted file mode 100644 index 762edc0f7e..0000000000 --- a/CHANGES/8821.misc +++ /dev/null @@ -1 +0,0 @@ -Properly utilize async in the content app, improving performance. diff --git a/CHANGES/8827.misc b/CHANGES/8827.misc deleted file mode 100644 index 11388f1141..0000000000 --- a/CHANGES/8827.misc +++ /dev/null @@ -1 +0,0 @@ -Refactored the sync pipeline to be simpler - merged ContentAssociation and ContentUnassociation stages. diff --git a/CHANGES/8867.bugfix b/CHANGES/8867.bugfix deleted file mode 100644 index 8cd7cc2d6d..0000000000 --- a/CHANGES/8867.bugfix +++ /dev/null @@ -1 +0,0 @@ -Retry downloads on ``ClientConnectorSSLError``, which appears to be spuriously returned by some CDNs. diff --git a/CHANGES/8868.bugfix b/CHANGES/8868.bugfix deleted file mode 100644 index bec7ea1324..0000000000 --- a/CHANGES/8868.bugfix +++ /dev/null @@ -1,9 +0,0 @@ -Fixed OpenAPI schema tag generation for resources that are nested more than 2 levels. - -This change is most evident in client libraries generated from the OpenAPI schema. - -Prior to this change, the API client for a resource located at -`/api/v3/pulp/exporters/core/pulp//exports/` was named `ExportersCoreExportsApi`. - -After this change, the API client for a resource located at -`/api/v3/pulp/exporters/core/pulp//exports/` is named `ExportersPulpExportsApi`. diff --git a/CHANGES/8869.bugfix b/CHANGES/8869.bugfix deleted file mode 100644 index 777ab765b6..0000000000 --- a/CHANGES/8869.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed request schema for ``/pulp/api/v3/repair/``, which did identify any arguments. This also fixes -the bindings. diff --git a/CHANGES/8876.deprecation b/CHANGES/8876.deprecation deleted file mode 100644 index 3f44c967db..0000000000 --- a/CHANGES/8876.deprecation +++ /dev/null @@ -1,2 +0,0 @@ -Deprecated the ``DELETE /pulp/api/v3/orphans/`` call. Instead use the -``POST /pulp/api/v3/orphans/cleanup/`` call. diff --git a/CHANGES/8881.feature b/CHANGES/8881.feature deleted file mode 100644 index b17569e83e..0000000000 --- a/CHANGES/8881.feature +++ /dev/null @@ -1 +0,0 @@ -Downloads from remote sources will now be retried on more kinds of errors, such as HTTP 500 or socket errors. diff --git a/CHANGES/8883.bugfix b/CHANGES/8883.bugfix deleted file mode 100644 index d8a305cff6..0000000000 --- a/CHANGES/8883.bugfix +++ /dev/null @@ -1 +0,0 @@ -Update default access policies in the database if they were unmodified by the administrator. diff --git a/CHANGES/8891.feature b/CHANGES/8891.feature deleted file mode 100644 index 73d934f2e4..0000000000 --- a/CHANGES/8891.feature +++ /dev/null @@ -1 +0,0 @@ -Add a correlation id filter to the task list endpoint. diff --git a/CHANGES/8897.feature b/CHANGES/8897.feature deleted file mode 100644 index a967fc6cc1..0000000000 --- a/CHANGES/8897.feature +++ /dev/null @@ -1 +0,0 @@ -Where before ``download_concurrency`` would previously be set to a default value upon creation, it will now be set NULL (but a default value will still be used). diff --git a/CHANGES/8926.bugfix b/CHANGES/8926.bugfix deleted file mode 100644 index 56fefb9647..0000000000 --- a/CHANGES/8926.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Pinning to psycopg2 < 2.9 as psycopg 2.9 doesn't work with django 2.2. More info at -https://github.com/django/django/commit/837ffcfa681d0f65f444d881ee3d69aec23770be. diff --git a/CHANGES/8930.feature b/CHANGES/8930.feature deleted file mode 100644 index d7b67c552c..0000000000 --- a/CHANGES/8930.feature +++ /dev/null @@ -1 +0,0 @@ -Added graceful shutdown to pulpcore workers. diff --git a/CHANGES/8946.doc b/CHANGES/8946.doc deleted file mode 100644 index a63e9b75ba..0000000000 --- a/CHANGES/8946.doc +++ /dev/null @@ -1 +0,0 @@ -Updated the docs.pulpproject.org to provide some immediate direction for better user orientation. diff --git a/CHANGES/8947.doc b/CHANGES/8947.doc deleted file mode 100644 index 53db508dfe..0000000000 --- a/CHANGES/8947.doc +++ /dev/null @@ -1 +0,0 @@ -Separated hardware and Filesystem information from the Architecture section and added them to the Installation section. diff --git a/CHANGES/8948.feature b/CHANGES/8948.feature deleted file mode 100644 index da3ec56b13..0000000000 --- a/CHANGES/8948.feature +++ /dev/null @@ -1,6 +0,0 @@ -Activate the new task worker type by default. - -.. warning:: - - If you intend to stick with the old tasking system, you should configure the - ``USE_NEW_WORKER_TYPE`` setting to false before upgrading. diff --git a/CHANGES/8949.doc b/CHANGES/8949.doc deleted file mode 100644 index 6edad2d8bd..0000000000 --- a/CHANGES/8949.doc +++ /dev/null @@ -1 +0,0 @@ -Added sub-headings and simplified language of Pulp concept section. diff --git a/CHANGES/8975.misc b/CHANGES/8975.misc deleted file mode 100644 index 7c6b9eb77a..0000000000 --- a/CHANGES/8975.misc +++ /dev/null @@ -1 +0,0 @@ -Some initial work for Django 3.0+ compat. diff --git a/CHANGES/8980.bugfix b/CHANGES/8980.bugfix deleted file mode 100644 index 81f59ca112..0000000000 --- a/CHANGES/8980.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed bug where artifacts and content were not always saved in Pulp with each -on_demand request serviced by content app. diff --git a/CHANGES/plugin_api/8480.deprecation b/CHANGES/plugin_api/8480.deprecation deleted file mode 100644 index aeee036920..0000000000 --- a/CHANGES/plugin_api/8480.deprecation +++ /dev/null @@ -1,2 +0,0 @@ -The ``pulpcore.plugin.viewsets.NewDistributionFilter`` is deprecated and will be removed from a -future release. Instead use ``pulpcore.plugin.viewsets.DistributionFilter``. diff --git a/CHANGES/plugin_api/8480.feature b/CHANGES/plugin_api/8480.feature deleted file mode 100644 index a73bfb7b15..0000000000 --- a/CHANGES/plugin_api/8480.feature +++ /dev/null @@ -1,2 +0,0 @@ -Added the ``pulpcore.plugin.viewsets.DistributionFilter``. This should be used instead of -``pulpcore.plugin.viewsets.NewDistributionFilter``. diff --git a/CHANGES/plugin_api/8487.feature b/CHANGES/plugin_api/8487.feature deleted file mode 100644 index 22c672d33a..0000000000 --- a/CHANGES/plugin_api/8487.feature +++ /dev/null @@ -1 +0,0 @@ -Added ``user_hidden`` field to ``Repository`` to hide repositories from users. diff --git a/CHANGES/plugin_api/8501.deprecation b/CHANGES/plugin_api/8501.deprecation deleted file mode 100644 index efe6b7669f..0000000000 --- a/CHANGES/plugin_api/8501.deprecation +++ /dev/null @@ -1,2 +0,0 @@ -Deprecate the use of the `reserved_resources_record__resource` in favor of `reserved_resources_record__contains`. -Tentative removal release is pulpcore==3.15. diff --git a/CHANGES/plugin_api/8501.removal b/CHANGES/plugin_api/8501.removal deleted file mode 100644 index 23dfcb46d1..0000000000 --- a/CHANGES/plugin_api/8501.removal +++ /dev/null @@ -1,2 +0,0 @@ -The usage of non-JSON serializable types of ``args`` and ``kwargs`` to tasks is no longer supported. -``uuid.UUID`` objects however will silently be converted to ``str``. diff --git a/CHANGES/plugin_api/8729.removal b/CHANGES/plugin_api/8729.removal deleted file mode 100644 index 9f3febebba..0000000000 --- a/CHANGES/plugin_api/8729.removal +++ /dev/null @@ -1,3 +0,0 @@ -Removed the ``versions_containing_content`` method from the -`pulpcore.plugin.models.RepositoryVersion`` object. Instead use -``RepositoryVersion.objects.with_content()``. diff --git a/CHANGES/plugin_api/8823.feature b/CHANGES/plugin_api/8823.feature deleted file mode 100644 index e6c089dbbc..0000000000 --- a/CHANGES/plugin_api/8823.feature +++ /dev/null @@ -1,5 +0,0 @@ -Added a ``timestamp_of_interest`` field to Content and Artifacts. This field can be updated by -calling a new method ``touch()`` on Artifacts and Content. Plugin writers should call this method -whenever they deal with Content or Artifacts. For example, this includes places where Content is -uploaded or added to Repository Versions. This will prevent Content and Artifacts from being cleaned -up when orphan cleanup becomes a non-blocking task in pulpcore 3.15. diff --git a/CHANGES/plugin_api/8827.removal b/CHANGES/plugin_api/8827.removal deleted file mode 100644 index 2fe14e9851..0000000000 --- a/CHANGES/plugin_api/8827.removal +++ /dev/null @@ -1 +0,0 @@ -Removed `pulpcore.plugin.stages.ContentUnassociation` from the plugin API. diff --git a/CHANGES/plugin_api/8844.feature b/CHANGES/plugin_api/8844.feature deleted file mode 100644 index 1e6a6dc340..0000000000 --- a/CHANGES/plugin_api/8844.feature +++ /dev/null @@ -1 +0,0 @@ -Exposed ``AsyncUpdateMixin`` through ``pulpcore.plugin.viewsets``. diff --git a/CHANGES/plugin_api/8868.bugfix b/CHANGES/plugin_api/8868.bugfix deleted file mode 100644 index bec7ea1324..0000000000 --- a/CHANGES/plugin_api/8868.bugfix +++ /dev/null @@ -1,9 +0,0 @@ -Fixed OpenAPI schema tag generation for resources that are nested more than 2 levels. - -This change is most evident in client libraries generated from the OpenAPI schema. - -Prior to this change, the API client for a resource located at -`/api/v3/pulp/exporters/core/pulp//exports/` was named `ExportersCoreExportsApi`. - -After this change, the API client for a resource located at -`/api/v3/pulp/exporters/core/pulp//exports/` is named `ExportersPulpExportsApi`. diff --git a/CHANGES/plugin_api/8881.deprecation b/CHANGES/plugin_api/8881.deprecation deleted file mode 100644 index 143ae2c44b..0000000000 --- a/CHANGES/plugin_api/8881.deprecation +++ /dev/null @@ -1 +0,0 @@ -Plugin writers who create custom downloaders by subclassing ``HttpDownloader`` no longer need to wrap the ``_run()`` method with a ``backoff`` decorator. Consequntly the ``http_giveup`` handler the sake of the ``backoff`` decorator is no longer needed and has been deprecated. It is likely to be removed in pulpcore 3.15. diff --git a/CHANGES/plugin_api/8881.feature b/CHANGES/plugin_api/8881.feature deleted file mode 100644 index 598a674114..0000000000 --- a/CHANGES/plugin_api/8881.feature +++ /dev/null @@ -1 +0,0 @@ -Added a field ``DEFAULT_MAX_RETRIES`` to the ``Remote`` base class - plugin writers can override the default number of retries attempted when file downloads failed for each type of remote. The default value is 3. diff --git a/CHANGES/plugin_api/8897.feature b/CHANGES/plugin_api/8897.feature deleted file mode 100644 index c6fd81c4fe..0000000000 --- a/CHANGES/plugin_api/8897.feature +++ /dev/null @@ -1 +0,0 @@ -Added a field ``DEFAULT_DOWNLOAD_CONCURRENCY`` to the Remote base class - plugin writers can override the number of concurrent downloads for each type of remote. The default value is 10. From adbf7fb45629cf7d40b52c3999f9d97ab19d54a2 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 1 Jul 2021 14:24:17 +0000 Subject: [PATCH 03/79] Release 3.14.0 Redmine Query: https://pulp.plan.io/issues?set_filter=1&status_id=*&issue_id=8823,8876,8883,8897,8947,8881,8801,8501,8800,8805,8729,8487,8946,8975,8658,8891,8869,8480,8774,8949,8827,8637,8868,8867,8980,8844,6837,8670,8708,8948,8926,8803,8930,8821 Redmine Milestone: https://pulp.plan.io/versions/218.json [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 60eefb224a..4776e7e8c6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.0.dev +current_version = 3.14.0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index ee3a4c9d45..9356da0bb0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.0.dev" +version = "3.14.0" # The full version, including alpha/beta/rc tags. -release = "3.14.0.dev" +release = "3.14.0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 1c6736ae2b..ef5732f2e8 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.0.dev" + version = "3.14.0" def ready(self): super().ready() diff --git a/setup.py b/setup.py index cbbc0009ba..875cd1da65 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.0.dev", + version="3.14.0", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 55a1805564ceff8b93b9e2ab166705d33894103d Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 1 Jul 2021 14:24:17 +0000 Subject: [PATCH 04/79] Bump to 3.14.1.dev [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4776e7e8c6..fb27ec908f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.0 +current_version = 3.14.1.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 9356da0bb0..0a424d99d6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.0" +version = "3.14.1.dev" # The full version, including alpha/beta/rc tags. -release = "3.14.0" +release = "3.14.1.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index ef5732f2e8..5faeb2f331 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.0" + version = "3.14.1.dev" def ready(self): super().ready() diff --git a/setup.py b/setup.py index 875cd1da65..bf5938ad76 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.0", + version="3.14.1.dev", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From b59f145ae2994ec80aa6872e6ceebc12dab73f5d Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Thu, 1 Jul 2021 16:03:42 -0400 Subject: [PATCH 05/79] Fixes changelog for 3.14.0 [noissue] --- CHANGES.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6cdae1352f..d4184dc3cc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,7 +17,9 @@ Changelog .. towncrier release notes start -3.14.0 (2021-07-01)REST API +3.14.0 (2021-07-01) +=================== +REST API -------- Features From 76d92ecc18e0579a164241d883c0c48fc305865a Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Thu, 1 Jul 2021 16:05:49 -0400 Subject: [PATCH 06/79] Updates with latest plugin-template GHA actions config The change pins towncrier to 19.9.0 and also adds an optional parameter to the release workflow. The optional 'before_script' parameter allows custom bash code to be run before script.sh is run. This is useful for resolving docs issues after a release. [noissue] --- .github/workflows/release.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bbeda8882d..cec4dfe789 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,11 @@ on: release: description: "Release tag (e.g. 3.2.1)" required: true + before_script: + description: | + Bash code to run before script.sh is executed. This should only be used when re-running + a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst + required: false env: RELEASE_WORKFLOW: true @@ -37,7 +42,7 @@ jobs: - name: Install python dependencies run: | echo ::group::PYDEPS - pip install bandersnatch bump2version gitpython python-redmine towncrier wheel + pip install bandersnatch bump2version gitpython python-redmine towncrier==19.9.0 wheel echo ::endgroup:: - name: Configure Git with pulpbot name and email @@ -139,6 +144,10 @@ jobs: if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} run: .github/workflows/scripts/install_ruby_client.sh + - name: Additional before_script + run: ${{ github.event.inputs.before_script }} + shell: bash + - name: Script if: ${{ env.TEST != 'generate-bindings' }} run: .github/workflows/scripts/script.sh From 2415faad2238548d07ed13fef75be28bbfeb4a5d Mon Sep 17 00:00:00 2001 From: David Davis Date: Wed, 7 Jul 2021 12:52:26 -0400 Subject: [PATCH 07/79] Removes ambiguity from OpenAPI schema for Exports. This patch clearly defines the exported_resources as a list of URI strings. The implementation for the ExportedResourceSerializer is shared with CreatedResource serializer. They now both inherit from a new RelatedResourceField. backports: #9008 https://pulp.plan.io/issues/9008 fixes #9025 (cherry picked from commit f52ba80e41ebbb532a87b6aa50821911a01348ff) --- .ci/assets/bindings/test_bindings.rb | 13 ++++++++++ CHANGES/9025.bugfix | 2 ++ pulpcore/app/serializers/__init__.py | 1 + pulpcore/app/serializers/base.py | 36 +++++++++++++++++++++++++++- pulpcore/app/serializers/exporter.py | 25 ++++++++----------- pulpcore/app/serializers/task.py | 23 ++---------------- 6 files changed, 63 insertions(+), 37 deletions(-) create mode 100644 CHANGES/9025.bugfix diff --git a/.ci/assets/bindings/test_bindings.rb b/.ci/assets/bindings/test_bindings.rb index 33d2450b97..c02e7dccf3 100644 --- a/.ci/assets/bindings/test_bindings.rb +++ b/.ci/assets/bindings/test_bindings.rb @@ -28,6 +28,9 @@ @fileremotes_api = PulpFileClient::RemotesFileApi.new @tasks_api = PulpcoreClient::TasksApi.new @uploads_api = PulpcoreClient::UploadsApi.new +@exporters_api = PulpcoreClient::ExportersPulpApi.new +@exports_api = PulpcoreClient::ExportersPulpExportsApi.new + def monitor_task(task_href) @@ -120,6 +123,16 @@ def upload_file_in_chunks(file_path) repository_version_1 = @repoversions_api.read(created_resources[0]) +# Create an exporter +exporter = @exporters_api.create({name: 'foo48', path: '/tmp/foo', repositories:[file_repository.pulp_href]}) + +# Create an export +export_response = @exports_api.create(exporter.pulp_href, versions: [repository_version_1.pulp_href]) +created_resources = monitor_task(export_response.task) + +# List exports +exports = @exports_api.list(exporter.pulp_href) + # Create an artifact from a local file file_path = File.join(ENV['GITHUB_WORKSPACE'], '.ci/assets/bindings/test_bindings.rb') artifact = @artifacts_api.create(File.new(file_path)) diff --git a/CHANGES/9025.bugfix b/CHANGES/9025.bugfix new file mode 100644 index 0000000000..ae4ec4ed8d --- /dev/null +++ b/CHANGES/9025.bugfix @@ -0,0 +1,2 @@ +Removed ambiguity from the OpenAPI schema for Exports. The exported_resources are now a list of URI strings. +(backported from #9008) diff --git a/pulpcore/app/serializers/__init__.py b/pulpcore/app/serializers/__init__.py index 672b72e7f3..54233dae2d 100644 --- a/pulpcore/app/serializers/__init__.py +++ b/pulpcore/app/serializers/__init__.py @@ -11,6 +11,7 @@ NestedIdentityField, NestedRelatedField, RelatedField, + RelatedResourceField, ValidateFieldsMixin, validate_unknown_fields, ) diff --git a/pulpcore/app/serializers/base.py b/pulpcore/app/serializers/base.py index 029a401ea4..7e6940488d 100644 --- a/pulpcore/app/serializers/base.py +++ b/pulpcore/app/serializers/base.py @@ -14,7 +14,11 @@ ) from pulpcore.app.models import Label, Task -from pulpcore.app.util import get_view_name_for_model +from pulpcore.app.util import ( + get_view_name_for_model, + get_viewset_for_model, + get_request_without_query_params, +) log = getLogger(__name__) @@ -261,6 +265,36 @@ def get_url(self, obj, view_name, request, *args, **kwargs): return super().get_url(obj, view_name, request, *args, **kwargs) +class RelatedResourceField(RelatedField): + """RelatedResourceField when relating a Resource object models. + + This field should be used to relate a list of non-homogeneous resources. e.g.: + CreatedResource and ExportedResource models that store relationships to arbitrary + resources. + + Specific implementation requires the model to be defined in the Meta:. + """ + + def to_representation(self, data): + # If the content object was deleted + if data.content_object is None: + return None + try: + if not data.content_object.complete: + return None + except AttributeError: + pass + + # query parameters can be ignored because we are looking just for 'pulp_href'; still, + # we need to use the request object due to contextual references required by some + # serializers + request = get_request_without_query_params(self.context) + + viewset = get_viewset_for_model(data.content_object) + serializer = viewset.serializer_class(data.content_object, context={"request": request}) + return serializer.data.get("pulp_href") + + class DetailIdentityField(_DetailFieldMixin, serializers.HyperlinkedIdentityField): """IdentityField for use in the pulp_href field of Master/Detail Serializers diff --git a/pulpcore/app/serializers/exporter.py b/pulpcore/app/serializers/exporter.py index dde9784347..d0ef0c45d4 100644 --- a/pulpcore/app/serializers/exporter.py +++ b/pulpcore/app/serializers/exporter.py @@ -13,11 +13,10 @@ ExportRelatedField, ModelSerializer, RelatedField, + RelatedResourceField, RepositoryVersionRelatedField, ) -from pulpcore.app.util import get_viewset_for_model - class ExporterSerializer(ModelSerializer): """ @@ -65,17 +64,10 @@ class Meta: fields = ModelSerializer.Meta.fields + ("name",) -class ExportedResourcesField(serializers.ListField): - def to_representation(self, obj): - result = [] - exported_resources = obj.exported_resources.all() - for exported_resource in exported_resources: - viewset = get_viewset_for_model(exported_resource.content_object) - serializer = viewset.serializer_class( - exported_resource.content_object, context={"request": None} - ) - result.append(serializer.data.get("pulp_href")) - return result +class ExportedResourceField(RelatedResourceField): + class Meta: + model = models.ExportedResource + fields = [] class ExportSerializer(ModelSerializer): @@ -93,8 +85,11 @@ class ExportSerializer(ModelSerializer): allow_null=True, ) - exported_resources = ExportedResourcesField( - help_text=_("Resources that were exported."), source="*", read_only=True + exported_resources = ExportedResourceField( + help_text=_("Resources that were exported."), + many=True, + read_only=True, + view_name="None", # This is a polymorphic field. The serializer does not need a view name. ) params = serializers.JSONField( diff --git a/pulpcore/app/serializers/task.py b/pulpcore/app/serializers/task.py index ed3cc65d0b..9ad3b55778 100755 --- a/pulpcore/app/serializers/task.py +++ b/pulpcore/app/serializers/task.py @@ -9,32 +9,13 @@ ModelSerializer, ProgressReportSerializer, RelatedField, + RelatedResourceField, TaskGroupStatusCountField, ) from pulpcore.constants import TASK_STATES -from pulpcore.app.util import get_viewset_for_model, get_request_without_query_params -class CreatedResourceSerializer(RelatedField): - def to_representation(self, data): - # If the content object was deleted - if data.content_object is None: - return None - try: - if not data.content_object.complete: - return None - except AttributeError: - pass - - # query parameters can be ignored because we are looking just for 'pulp_href'; still, - # we need to use the request object due to contextual references required by some - # serializers - request = get_request_without_query_params(self.context) - - viewset = get_viewset_for_model(data.content_object) - serializer = viewset.serializer_class(data.content_object, context={"request": request}) - return serializer.data.get("pulp_href") - +class CreatedResourceSerializer(RelatedResourceField): class Meta: model = models.CreatedResource fields = [] From ddcb75e84f1cac1d3d400fa7051da196217f21f8 Mon Sep 17 00:00:00 2001 From: David Davis Date: Wed, 7 Jul 2021 12:53:22 -0400 Subject: [PATCH 08/79] Fix syncs of repos from file:// urls backports: #9003 https://pulp.plan.io/issues/9003 fixes #9015 (cherry picked from commit e4b17247d30b85841e5065d745a1fbb219a58cec) --- CHANGES/9015.bugfix | 2 ++ pulpcore/download/base.py | 2 ++ pulpcore/download/file.py | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 CHANGES/9015.bugfix diff --git a/CHANGES/9015.bugfix b/CHANGES/9015.bugfix new file mode 100644 index 0000000000..dcdb4713c8 --- /dev/null +++ b/CHANGES/9015.bugfix @@ -0,0 +1,2 @@ +Fixed a regression preventing syncs from file:// urls. +(backported from #9003) diff --git a/pulpcore/download/base.py b/pulpcore/download/base.py index 692b6f5532..7665aac62d 100644 --- a/pulpcore/download/base.py +++ b/pulpcore/download/base.py @@ -76,6 +76,8 @@ def __init__( expected_digests=None, expected_size=None, semaphore=None, + *args, + **kwargs, ): """ Create a BaseDownloader object. This is expected to be called by all subclasses. diff --git a/pulpcore/download/file.py b/pulpcore/download/file.py index fbb1adbbf1..a078e3b99d 100644 --- a/pulpcore/download/file.py +++ b/pulpcore/download/file.py @@ -19,7 +19,7 @@ class FileDownloader(BaseDownloader): :class:`~pulpcore.plugin.download.BaseDownloader` """ - def __init__(self, url, **kwargs): + def __init__(self, url, *args, **kwargs): """ Download files from a url that starts with `file://` @@ -37,7 +37,7 @@ def __init__(self, url, **kwargs): RemoteSerializer().validate_url(url) p = urlparse(url) self._path = os.path.abspath(os.path.join(p.netloc, p.path)) - super().__init__(url, **kwargs) + super().__init__(url, *args, **kwargs) async def _run(self, extra_data=None): """ From a494186ffa459dbbee4e30f47057f018976ffac7 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 7 Jul 2021 17:42:07 +0000 Subject: [PATCH 09/79] Building changelog for 3.14.1 [noissue] --- CHANGES.rst | 22 ++++++++++++++++++++++ CHANGES/9015.bugfix | 2 -- CHANGES/9025.bugfix | 2 -- 3 files changed, 22 insertions(+), 4 deletions(-) delete mode 100644 CHANGES/9015.bugfix delete mode 100644 CHANGES/9025.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index d4184dc3cc..a6bb5101a4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,28 @@ Changelog .. towncrier release notes start +3.14.1 (2021-07-07) +=================== +REST API +-------- + +Bugfixes +~~~~~~~~ + +- Fixed a regression preventing syncs from file:// urls. + (backported from #9003) + `#9015 `_ +- Removed ambiguity from the OpenAPI schema for Exports. The exported_resources are now a list of URI strings. + (backported from #9008) + `#9025 `_ + + +Plugin API +---------- + +No significant changes. + + 3.14.0 (2021-07-01) =================== REST API diff --git a/CHANGES/9015.bugfix b/CHANGES/9015.bugfix deleted file mode 100644 index dcdb4713c8..0000000000 --- a/CHANGES/9015.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a regression preventing syncs from file:// urls. -(backported from #9003) diff --git a/CHANGES/9025.bugfix b/CHANGES/9025.bugfix deleted file mode 100644 index ae4ec4ed8d..0000000000 --- a/CHANGES/9025.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Removed ambiguity from the OpenAPI schema for Exports. The exported_resources are now a list of URI strings. -(backported from #9008) From d88f28c562150255536f2ad1d2c95874e61c276f Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 7 Jul 2021 17:42:08 +0000 Subject: [PATCH 10/79] Release 3.14.1 Redmine Query: https://pulp.plan.io/issues?set_filter=1&status_id=*&issue_id=9015,9025 Redmine Milestone: https://pulp.plan.io/versions/257.json [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index fb27ec908f..80e0457fab 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.1.dev +current_version = 3.14.1 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 0a424d99d6..063aef41dd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.1.dev" +version = "3.14.1" # The full version, including alpha/beta/rc tags. -release = "3.14.1.dev" +release = "3.14.1" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 5faeb2f331..d6ceba6492 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.1.dev" + version = "3.14.1" def ready(self): super().ready() diff --git a/setup.py b/setup.py index bf5938ad76..b461457bfa 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.1.dev", + version="3.14.1", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 634ca5c95796d4dc1d3e0abb9ec9b61998a0d8b0 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 7 Jul 2021 17:42:08 +0000 Subject: [PATCH 11/79] Bump to 3.14.2.dev [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 80e0457fab..01cc8a2661 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.1 +current_version = 3.14.2.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 063aef41dd..5b5e0b743c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.1" +version = "3.14.2.dev" # The full version, including alpha/beta/rc tags. -release = "3.14.1" +release = "3.14.2.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index d6ceba6492..a5ee095eb5 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.1" + version = "3.14.2.dev" def ready(self): super().ready() diff --git a/setup.py b/setup.py index b461457bfa..ba5a374fd5 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.1", + version="3.14.2.dev", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 64176523d0bf899ebd32b8668d8ed6be7cc7af39 Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Mon, 12 Jul 2021 12:09:10 -0400 Subject: [PATCH 12/79] Updates GHA config to latest from plugin_template [noissue] --- .ci/ansible/Containerfile.j2 | 2 +- .ci/ansible/settings.py.j2 | 8 +++---- .ci/ansible/start_container.yaml | 3 ++- .ci/scripts/cherrypick.sh | 5 +++++ .github/workflows/nightly.yml | 22 +++++++++++++++++++ .github/workflows/scripts/install.sh | 2 ++ .../scripts/install_python_client.sh | 2 +- .../workflows/scripts/install_ruby_client.sh | 2 +- .../workflows/scripts/publish_plugin_pypi.sh | 2 -- .github/workflows/scripts/script.sh | 2 +- .../scripts/stage-changelog-for-master.py | 9 +++++++- template_config.yml | 1 + 12 files changed, 48 insertions(+), 12 deletions(-) diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index d6b3a8c03f..66c46cc40b 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,4 +1,4 @@ -FROM {{ ci_base | default("pulp/pulp-ci-centos:latest") }} +FROM {{ ci_base | default("pulp/pulp-ci-centos:" + pulp_container_tag) }} # Add source directories to container {% for item in plugins %} diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 3de1319294..e0e040a7ac 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -1,9 +1,9 @@ -CONTENT_ORIGIN = "http://pulp:80" -ANSIBLE_API_HOSTNAME = "http://pulp:80" -ANSIBLE_CONTENT_HOSTNAME = "http://pulp:80/pulp/content" +CONTENT_ORIGIN = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" +ANSIBLE_API_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" +ANSIBLE_CONTENT_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/pulp/content" PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem" PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem" -TOKEN_SERVER = "http://pulp:80/token/" +TOKEN_SERVER = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/token/" TOKEN_SIGNATURE_ALGORITHM = "ES256" {% if pulp_settings %} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index d75a7083a3..8661bb88d1 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -71,7 +71,8 @@ - name: "Wait for Pulp" uri: url: "http://pulp/pulp/api/v3/status/" - follow_redirects: none + follow_redirects: all + validate_certs: no register: result until: result.status == 200 retries: 12 diff --git a/.ci/scripts/cherrypick.sh b/.ci/scripts/cherrypick.sh index 3e8e4d872a..ebc8d8f1d6 100755 --- a/.ci/scripts/cherrypick.sh +++ b/.ci/scripts/cherrypick.sh @@ -9,6 +9,11 @@ set -e +if [ ! -d CHANGES ]; then + echo "Error: no CHANGES directory detected. This script must be run from the project root." + exit 1 +fi + if [ $# -lt 3 ] then echo "Usage: .ci/scripts/cherrypick.sh [commit-hash] [original-issue-id] [backport-issue-id]" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 28566f746e..933b274a6b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -102,6 +102,17 @@ jobs: name: docs.tar path: docs/docs.tar + - name: After failure + if: failure() + run: | + http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp pip3 list + publish: runs-on: ubuntu-latest needs: test @@ -201,3 +212,14 @@ jobs: tar -xvf docs.tar -C ./docs .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} + + - name: After failure + if: failure() + run: | + http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp pip3 list diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 2f8e9fabf4..4053cfe2df 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -62,6 +62,8 @@ VARSYAML cat >> vars/main.yaml << VARSYAML pulp_settings: {"allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"]} +pulp_scheme: http +pulp_container_tag: latest VARSYAML if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "upgrade" || "$TEST" == "s3" || "$TEST" == "plugin-from-pypi" ]]; then diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index 31b9085834..da768052ba 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -16,7 +16,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. pip install twine wheel -export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin core --arg legacy_plugin pulpcore -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') +export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin core --arg legacy_plugin pulpcore -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then export VERSION=${REPORTED_VERSION} diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh index 6d7cedab46..078b19bdc0 100755 --- a/.github/workflows/scripts/install_ruby_client.sh +++ b/.github/workflows/scripts/install_ruby_client.sh @@ -14,7 +14,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. export PULP_URL="${PULP_URL:-http://pulp}" -export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin core --arg legacy_plugin pulpcore -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') +export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin core --arg legacy_plugin pulpcore -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then export VERSION=${REPORTED_VERSION} diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index e889ae84a6..c67c2f571f 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -12,8 +12,6 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. set -euv -export PULP_URL="${PULP_URL:-http://pulp}" - export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulpcore/$1/) if [ "$response" == "200" ]; then diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 3900802c90..c2ec615c25 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -46,7 +46,7 @@ if [[ "$TEST" = "docs" ]]; then fi if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin core --arg legacy_plugin pulpcore -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') + REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin core --arg legacy_plugin pulpcore -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulpcore/$REPORTED_VERSION/) if [ "$response" == "200" ]; then diff --git a/.github/workflows/scripts/stage-changelog-for-master.py b/.github/workflows/scripts/stage-changelog-for-master.py index 1ff621c728..0c344567ca 100755 --- a/.github/workflows/scripts/stage-changelog-for-master.py +++ b/.github/workflows/scripts/stage-changelog-for-master.py @@ -10,6 +10,7 @@ import textwrap from git import Repo +from git.exc import GitCommandError helper = textwrap.dedent( @@ -53,5 +54,11 @@ git = repo.git git.stash() git.checkout("origin/master") -git.cherry_pick(changelog_commit.hexsha) +try: + git.cherry_pick(changelog_commit.hexsha) +except GitCommandError: + git.add("CHANGES/") + # Don't try opening an editor for the commit message + with git.custom_environment(GIT_EDITOR="true"): + git.cherry_pick("--continue") git.reset("origin/master") diff --git a/template_config.yml b/template_config.yml index 6cbd216686..b64cba52b8 100644 --- a/template_config.yml +++ b/template_config.yml @@ -33,6 +33,7 @@ plugin_default_branch: master plugin_name: pulpcore plugin_snake: pulpcore publish_docs_to_pulpprojectdotorg: true +pulp_scheme: http pulp_settings: allowed_export_paths: - /tmp From 8f44267353b8322f2458d255c068d5392d9a7241 Mon Sep 17 00:00:00 2001 From: David Davis Date: Tue, 13 Jul 2021 11:59:27 -0400 Subject: [PATCH 13/79] Use proxy auth from the Remote config. backports: #9024 https://pulp.plan.io/issues/9024 fixes #9068 (cherry picked from commit 0c9541d3699ea54eb25509f6217002c3a4fc7576) --- CHANGES/9068.bugfix | 2 ++ pulpcore/download/http.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 CHANGES/9068.bugfix diff --git a/CHANGES/9068.bugfix b/CHANGES/9068.bugfix new file mode 100644 index 0000000000..6e9f2c31d3 --- /dev/null +++ b/CHANGES/9068.bugfix @@ -0,0 +1,2 @@ +Use proxy auth from Remote config to download content from a remote repository. +(backported from #9024) diff --git a/pulpcore/download/http.py b/pulpcore/download/http.py index 6822e612a2..9017550536 100644 --- a/pulpcore/download/http.py +++ b/pulpcore/download/http.py @@ -273,7 +273,9 @@ async def _run(self, extra_data=None): """ if self.download_throttler: await self.download_throttler.acquire() - async with self.session.get(self.url, proxy=self.proxy, auth=self.auth) as response: + async with self.session.get( + self.url, proxy=self.proxy, proxy_auth=self.proxy_auth, auth=self.auth + ) as response: self.raise_for_status(response) to_return = await self._handle_response(response) await response.release() From c28d2664f743552cb9ef3f24102361349e0df509 Mon Sep 17 00:00:00 2001 From: David Davis Date: Tue, 13 Jul 2021 12:03:13 -0400 Subject: [PATCH 14/79] Make auto-distribute feature more intuitive backports: #9039 https://pulp.plan.io/issues/9039 fixes #9059 (cherry picked from commit 2cc98040f323530245f198486c9a28f81839f31a) --- CHANGES/9059.bugfix | 2 ++ pulpcore/app/models/publication.py | 11 ++++++++++- pulpcore/content/handler.py | 10 ++++++++-- 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 CHANGES/9059.bugfix diff --git a/CHANGES/9059.bugfix b/CHANGES/9059.bugfix new file mode 100644 index 0000000000..d551291da5 --- /dev/null +++ b/CHANGES/9059.bugfix @@ -0,0 +1,2 @@ +Fixed the behavior of setting "repository" on a distribution for publication-based plugins. +(backported from #9039) diff --git a/pulpcore/app/models/publication.py b/pulpcore/app/models/publication.py index cfb55a10c0..28a8c3ad4d 100644 --- a/pulpcore/app/models/publication.py +++ b/pulpcore/app/models/publication.py @@ -125,13 +125,14 @@ def delete(self, **kwargs): Deletes the Task.created_resource when complete is False. """ with transaction.atomic(): + # invalidate cache if settings.CACHE_ENABLED: # Find any publications being served directly base_paths = self.distribution_set.values_list("base_path", flat=True) # Find any publications being served indirectly by auto-distribute feature versions = self.repository.versions.all() pubs = Publication.objects.filter(repository_version__in=versions, complete=True) - publication = pubs.latest("repository_version", "-pulp_created") + publication = pubs.latest("repository_version", "pulp_created") if self.pk == publication.pk: base_paths |= self.repository.distributions.values_list("base_path", flat=True) # Invalidate cache for all distributions serving this publication @@ -183,6 +184,14 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.delete() raise + # invalidate cache + if settings.CACHE_ENABLED: + base_paths = Distribution.objects.filter( + repository=self.repository_version.repository + ).values_list("base_path", flat=True) + if base_paths: + Cache().delete(base_key=base_paths) + class PublishedArtifact(BaseModel): """ diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py index 55d98db41c..eba92e21e3 100644 --- a/pulpcore/content/handler.py +++ b/pulpcore/content/handler.py @@ -466,7 +466,7 @@ def check_permit_blocking(): def get_latest_publication_or_version_blocking(): nonlocal repo_version nonlocal publication - repo_version = repository.latest_version() + # Search for publication serving the closest latest version if not publication: try: @@ -474,10 +474,16 @@ def get_latest_publication_or_version_blocking(): publications = Publication.objects.filter( repository_version__in=versions, complete=True ) - publication = publications.latest("repository_version", "-pulp_created") + publication = publications.select_related("repository_version").latest( + "repository_version", "pulp_created" + ) + repo_version = publication.repository_version except ObjectDoesNotExist: pass + if not repo_version: + repo_version = repository.latest_version() + await loop.run_in_executor(None, get_latest_publication_or_version_blocking) if publication: From 8c668c26b4ecee71a9dc137dcabd2e6b787203d6 Mon Sep 17 00:00:00 2001 From: David Davis Date: Tue, 13 Jul 2021 12:03:56 -0400 Subject: [PATCH 15/79] Replace DictField with JSONField backports #8954 fixes #9058 (cherry picked from commit 3bc707ad6aab45d76f06abab7ed0870049cc28f8) --- CHANGES/9058.bugfix | 2 ++ pulpcore/app/serializers/fields.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 CHANGES/9058.bugfix diff --git a/CHANGES/9058.bugfix b/CHANGES/9058.bugfix new file mode 100644 index 0000000000..3ffcee15f1 --- /dev/null +++ b/CHANGES/9058.bugfix @@ -0,0 +1,2 @@ +Fixed a bug that caused a serializer to ignore form data for ``pulp_labels``. +(backported from #8954) diff --git a/pulpcore/app/serializers/fields.py b/pulpcore/app/serializers/fields.py index 6235baebe0..7ceac49847 100644 --- a/pulpcore/app/serializers/fields.py +++ b/pulpcore/app/serializers/fields.py @@ -377,8 +377,8 @@ def get_attribute(self, instance): return instance.tasks.filter(state=self.state).count() -class LabelsField(serializers.DictField): - """Serializer field for pulp_labels.""" +class LabelsField(serializers.JSONField): + """A serializer field for pulp_labels.""" def to_representation(self, labels): """ @@ -397,14 +397,16 @@ def to_internal_value(self, data): Ensures that data conforms to key/value dict. Args: - data (dict): A dictionary that maps label key to label value + data (str or dict): A dictionary which maps a label key to a label value Returns: A validated data dict for labels mapping keys to values Raises: - rest_framework.serializers.ValidationError: if data is invalid (eg not a dict) + rest_framework.serializers.ValidationError: if data is invalid (e.g., not a dict) """ + data = super().to_internal_value(data) + if not isinstance(data, dict): raise serializers.ValidationError(_("Data must be supplied as a key/value hash.")) for key, value in data.items(): From b1831282a2968f4541a7decd39f1a07bb2c14e4a Mon Sep 17 00:00:00 2001 From: David Davis Date: Tue, 13 Jul 2021 12:04:39 -0400 Subject: [PATCH 16/79] Adds Range header support to content app This unskips the `Range` header and adds support for it to the content app. backports #8865 fixes #9057 (cherry picked from commit df2c8f109ec14e6bdb204058eb575d62fb58ad1a) --- CHANGES/9057.bugfix | 4 + pulpcore/content/handler.py | 45 +++++- .../api/using_plugin/test_distributions.py | 141 +++++++++++++++--- 3 files changed, 163 insertions(+), 27 deletions(-) create mode 100644 CHANGES/9057.bugfix diff --git a/CHANGES/9057.bugfix b/CHANGES/9057.bugfix new file mode 100644 index 0000000000..fdb8dc2594 --- /dev/null +++ b/CHANGES/9057.bugfix @@ -0,0 +1,4 @@ +Fixed bug where content app would not respond to ``Range`` HTTP Header in requests when +``remote.policy`` was either ``on_demand`` or ``streamed``. For example this request is used by +Anaconda clients. +(backported from #8865) diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py index eba92e21e3..2f896ca6a7 100644 --- a/pulpcore/content/handler.py +++ b/pulpcore/content/handler.py @@ -671,7 +671,7 @@ def get_remote_artifacts_blocking(): return response except (ClientResponseError, UnsupportedDigestValidationError) as e: - log.warn( + log.warning( _("Could not download remote artifact at '{}': {}").format( remote_artifact.url, str(e) ) @@ -815,15 +815,50 @@ def cast_remote_blocking(): remote = await loop.run_in_executor(None, cast_remote_blocking) - async def handle_headers(headers): + range_start, range_stop = request.http_range.start, request.http_range.stop + if range_start or range_stop: + response.set_status(206) + + async def handle_response_headers(headers): for name, value in headers.items(): - if name.lower() in self.hop_by_hop_headers: + lower_name = name.lower() + if lower_name in self.hop_by_hop_headers: continue + + if response.status == 206 and lower_name == "content-length": + range_bytes = int(value) + start = 0 if range_start is None else range_start + stop = range_bytes if range_stop is None else range_stop + + range_bytes = range_bytes - range_start + range_bytes = range_bytes - (int(value) - stop) + response.headers[name] = str(range_bytes) + + response.headers["Content-Range"] = "bytes {0}-{1}/{2}".format( + start, stop - start + 1, int(value) + ) + continue + response.headers[name] = value await response.prepare(request) + data_size_handled = 0 + async def handle_data(data): - await response.write(data) + nonlocal data_size_handled + if range_start or range_stop: + start_byte_pos = 0 + end_byte_pos = len(data) + if range_start: + start_byte_pos = max(0, range_start - data_size_handled) + if range_stop: + end_byte_pos = min(len(data), range_stop - data_size_handled) + + data_for_client = data[start_byte_pos:end_byte_pos] + await response.write(data_for_client) + data_size_handled = data_size_handled + len(data) + else: + await response.write(data) if remote.policy != Remote.STREAMED: await original_handle_data(data) @@ -832,7 +867,7 @@ async def finalize(): await original_finalize() downloader = remote.get_downloader( - remote_artifact=remote_artifact, headers_ready_callback=handle_headers + remote_artifact=remote_artifact, headers_ready_callback=handle_response_headers ) original_handle_data = downloader.handle_data downloader.handle_data = handle_data diff --git a/pulpcore/tests/functional/api/using_plugin/test_distributions.py b/pulpcore/tests/functional/api/using_plugin/test_distributions.py index 3ae9903950..10210ee0b3 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_distributions.py +++ b/pulpcore/tests/functional/api/using_plugin/test_distributions.py @@ -4,10 +4,11 @@ import unittest from urllib.parse import urljoin -from pulp_smash import api, config, utils +from pulp_smash import api, cli, config, utils from pulp_smash.pulp3.bindings import delete_orphans from pulp_smash.pulp3.utils import ( download_content_unit, + download_content_unit_return_requests_response, gen_distribution, gen_repo, get_content, @@ -329,24 +330,113 @@ def test_content_served_immediate(self): self.setup_download_test("immediate") self.do_test_content_served() - @unittest.skip("https://pulp.plan.io/issues/8865") - def test_content_served_on_demand_with_range_request(self): - """Assert that on_demand content can be properly downloaded with range requests.""" - self.setup_download_test("on_demand") - self.do_range_request_download_test() + def test_content_served_streamed(self): + """Assert that streamed content can be properly downloaded.""" + self.setup_download_test("streamed") + self.do_test_content_served() - def test_content_served_immediate_with_range_request(self): + def test_content_served_immediate_with_range_request_inside_one_chunk(self): """Assert that downloaded content can be properly downloaded with range requests.""" - self.setup_download_test("immediate") - self.do_range_request_download_test() + self.setup_download_test( + "immediate", url="https://fixtures.pulpproject.org/file-chunked/PULP_MANIFEST" + ) + range_headers = {"Range": "bytes=1048586-1049586"} + num_bytes = 1001 + self.do_range_request_download_test(range_headers, num_bytes) + + def test_content_served_immediate_with_range_request_over_three_chunks(self): + """Assert that downloaded content can be properly downloaded with range requests.""" + self.setup_download_test( + "immediate", url="https://fixtures.pulpproject.org/file-chunked/PULP_MANIFEST" + ) + range_headers = {"Range": "bytes=1048176-2248576"} + num_bytes = 1200401 + self.do_range_request_download_test(range_headers, num_bytes) + + def test_content_served_on_demand_with_range_request_over_three_chunks(self): + """Assert that on_demand content can be properly downloaded with range requests.""" + self.setup_download_test( + "on_demand", url="https://fixtures.pulpproject.org/file-chunked/PULP_MANIFEST" + ) + range_headers = {"Range": "bytes=1048176-2248576"} + num_bytes = 1200401 + self.do_range_request_download_test(range_headers, num_bytes) + + def test_content_served_streamed_with_range_request_over_three_chunks(self): + """Assert that streamed content can be properly downloaded with range requests.""" + self.setup_download_test( + "streamed", url="https://fixtures.pulpproject.org/file-chunked/PULP_MANIFEST" + ) + range_headers = {"Range": "bytes=1048176-2248576"} + num_bytes = 1200401 + self.do_range_request_download_test(range_headers, num_bytes) + + def test_content_served_immediate_with_multiple_different_range_requests(self): + """Assert that multiple requests with different Range header values work as expected.""" + self.setup_download_test( + "immediate", url="https://fixtures.pulpproject.org/file-chunked/PULP_MANIFEST" + ) + range_headers = {"Range": "bytes=1048176-2248576"} + num_bytes = 1200401 + self.do_range_request_download_test(range_headers, num_bytes) + range_headers = {"Range": "bytes=2042176-3248576"} + num_bytes = 1206401 + self.do_range_request_download_test(range_headers, num_bytes) + + def test_content_served_immediate_with_range_request_invalid_start_value(self): + """Assert that range requests with a negative start value errors as expected.""" + cfg = config.get_config() + cli_client = cli.Client(cfg) + storage = utils.get_pulp_setting(cli_client, "DEFAULT_FILE_STORAGE") + if storage != "pulpcore.app.models.storage.FileSystem": + self.skipTest("The S3 test API project doesn't handle invalid Range values correctly") + self.setup_download_test( + "immediate", url="https://fixtures.pulpproject.org/file-chunked/PULP_MANIFEST" + ) - def setup_download_test(self, policy): + self.assertRaises( + HTTPError, + download_content_unit_return_requests_response, + self.cfg, + self.distribution.to_dict(), + "1.iso", + headers={"Range": "bytes=-1-11"}, + ) + + def test_content_served_immediate_with_range_request_too_large_end_value(self): + """Assert that a range request with a end value that is larger than the data works still.""" + self.setup_download_test( + "immediate", url="https://fixtures.pulpproject.org/file-chunked/PULP_MANIFEST" + ) + range_headers = {"Range": "bytes=10485260-10485960"} + num_bytes = 500 + self.do_range_request_download_test(range_headers, num_bytes) + + def test_content_served_immediate_with_range_request_start_value_larger_than_content(self): + """Assert that a range request with a start value larger than the content errors.""" + self.setup_download_test( + "immediate", url="https://fixtures.pulpproject.org/file-chunked/PULP_MANIFEST" + ) + self.assertRaises( + HTTPError, + download_content_unit_return_requests_response, + self.cfg, + self.distribution.to_dict(), + "1.iso", + headers={"Range": "bytes=10485860-10485870"}, + ) + + def setup_download_test(self, policy, url=None): # Create a repository self.repo = self.repo_api.create(gen_repo()) self.addCleanup(self.repo_api.delete, self.repo.pulp_href) # Create a remote - self.remote = self.remote_api.create(gen_file_remote(policy=policy)) + remote_options = {"policy": policy} + if url: + remote_options["url"] = url + + self.remote = self.remote_api.create(gen_file_remote(**remote_options)) self.addCleanup(self.remote_api.delete, self.remote.pulp_href) # Sync the repository. @@ -388,19 +478,26 @@ def do_test_content_served(self): self.assertEqual(len(pulp_manifest), FILE_FIXTURE_COUNT, pulp_manifest) - def do_range_request_download_test(self): + def do_range_request_download_test(self, range_header, expected_bytes): file_path = "1.iso" - headers = {"Range": "bytes=0-9"} # first 10 bytes - NUM_BYTES = 10 - - req1 = download_content_unit( - self.cfg, self.distribution.to_dict(), file_path, headers=headers + req1_reponse = download_content_unit_return_requests_response( + self.cfg, self.distribution.to_dict(), file_path, headers=range_header ) - req2 = download_content_unit( - self.cfg, self.distribution.to_dict(), file_path, headers=headers + req2_response = download_content_unit_return_requests_response( + self.cfg, self.distribution.to_dict(), file_path, headers=range_header ) - self.assertEqual(NUM_BYTES, len(req1)) - self.assertEqual(NUM_BYTES, len(req2)) - self.assertEqual(req1, req2) + self.assertEqual(expected_bytes, len(req1_reponse.content)) + self.assertEqual(expected_bytes, len(req2_response.content)) + self.assertEqual(req1_reponse.content, req2_response.content) + + self.assertEqual(req1_reponse.status_code, 206) + self.assertEqual(req1_reponse.status_code, req2_response.status_code) + + self.assertEqual(str(expected_bytes), req1_reponse.headers["Content-Length"]) + self.assertEqual(str(expected_bytes), req2_response.headers["Content-Length"]) + + self.assertEqual( + req1_reponse.headers["Content-Range"], req2_response.headers["Content-Range"] + ) From 10152ee4726f8421259854741b6004f235a07af7 Mon Sep 17 00:00:00 2001 From: David Davis Date: Tue, 13 Jul 2021 14:48:35 -0400 Subject: [PATCH 17/79] Fix cache error when getting invalid files backports: #9074 fixes #9077 (cherry picked from commit ca0ef8ff68bc4c068e45f53552f5077994bfcdba) --- CHANGES/9077.bugfix | 2 ++ pulpcore/cache/cache.py | 4 +++ .../api/using_plugin/test_content_cache.py | 25 +++++++++++++------ pulpcore/tests/unit/test_cache.py | 1 + 4 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 CHANGES/9077.bugfix diff --git a/CHANGES/9077.bugfix b/CHANGES/9077.bugfix new file mode 100644 index 0000000000..b98bba2737 --- /dev/null +++ b/CHANGES/9077.bugfix @@ -0,0 +1,2 @@ +Fixed server error when accessing invalid files from content app base directory +(backported from #9074) diff --git a/pulpcore/cache/cache.py b/pulpcore/cache/cache.py index 29ff484256..2d8c242268 100644 --- a/pulpcore/cache/cache.py +++ b/pulpcore/cache/cache.py @@ -45,6 +45,8 @@ def set(self, key, value, expires=None, base_key=None): def exists(self, key=None, base_key=None): """Checks if cached entries exist""" + if not base_key and base_key is not None: + return False # Failsafe for passing empty list/str base_key = base_key or self.default_base_key if key: return self.redis.hexists(base_key, key) @@ -97,6 +99,8 @@ async def set(self, key, value, expires=None, base_key=None): async def exists(self, key=None, base_key=None): """Checks if cached entries exist""" + if not base_key and base_key is not None: + return False # Failsafe for passing empty list/str base_key = base_key or self.default_base_key if key: return await self.redis.hexists(base_key, key) diff --git a/pulpcore/tests/functional/api/using_plugin/test_content_cache.py b/pulpcore/tests/functional/api/using_plugin/test_content_cache.py index 2cfcb6b37e..1c9a1f66b3 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_content_cache.py +++ b/pulpcore/tests/functional/api/using_plugin/test_content_cache.py @@ -72,8 +72,8 @@ def test_02_remove_repository_invalidates(self): body = PatchedfileFileDistribution(repository="") monitor_task(self.dis_api.partial_update(self.distro.pulp_href, body).task) files = ["", "PULP_MANIFEST", "1.iso"] - for i, file in enumerate(files): - self.assertEqual((404, None), self.check_cache(file)) + for file in files: + self.assertEqual((404, None), self.check_cache(file), file) def test_03_restore_repository(self): """Checks that responses are cacheable after repository is added back""" @@ -90,7 +90,9 @@ def test_04_multiple_distributions(self): url = urljoin(PULP_CONTENT_BASE_URL, f"{self.distro2[0].base_path}/") files = ["", "", "PULP_MANIFEST", "PULP_MANIFEST", "1.iso", "1.iso"] for i, file in enumerate(files): - self.assertEqual((200, "HIT" if i % 2 == 1 else "MISS"), self.check_cache(file, url)) + self.assertEqual( + (200, "HIT" if i % 2 == 1 else "MISS"), self.check_cache(file, url), file + ) def test_05_invalidate_multiple_distributions(self): """Test that updating a repository pointed by multiple distributions invalidates all""" @@ -104,14 +106,16 @@ def test_05_invalidate_multiple_distributions(self): files = ["", "", "PULP_MANIFEST", "PULP_MANIFEST", "2.iso", "2.iso"] for i, file in enumerate(files): self.assertEqual((200, "HIT" if i % 2 == 1 else "MISS"), self.check_cache(file), file) - self.assertEqual((200, "HIT" if i % 2 == 1 else "MISS"), self.check_cache(file, url)) + self.assertEqual( + (200, "HIT" if i % 2 == 1 else "MISS"), self.check_cache(file, url), file + ) def test_06_delete_distribution_invalidates_one(self): """Tests that deleting one distribution sharing a repository only invalidates its cache""" url = urljoin(PULP_CONTENT_BASE_URL, f"{self.distro2[0].base_path}/") monitor_task(self.dis_api.delete(self.distro2[0].pulp_href).task) files = ["", "PULP_MANIFEST", "2.iso"] - for i, file in enumerate(files): + for file in files: self.assertEqual((200, "HIT"), self.check_cache(file), file) self.assertEqual((404, None), self.check_cache(file, url), file) @@ -119,7 +123,7 @@ def test_07_delete_extra_pub_doesnt_invalidate(self): """Test that deleting a publication not being served doesn't invalidate cache""" self.pub_api.delete(self.pub2.pulp_href) files = ["", "PULP_MANIFEST", "2.iso"] - for i, file in enumerate(files): + for file in files: self.assertEqual((200, "HIT"), self.check_cache(file), file) def test_08_delete_served_pub_does_invalidate(self): @@ -134,9 +138,16 @@ def test_09_delete_repo_invalidates(self): """Tests that deleting a repository invalidates the cache""" monitor_task(self.repo_api.delete(self.repo.pulp_href).task) files = ["", "PULP_MANIFEST", "2.iso"] - for i, file in enumerate(files): + for file in files: self.assertEqual((404, None), self.check_cache(file), file) + def test_10_no_error_when_accessing_invalid_file(self): + """Tests that accessing a file that doesn't exist on content app gives 404""" + files = ["invalid", "another/bad-one", "DNE/"] + url = PULP_CONTENT_BASE_URL + for file in files: + self.assertEqual((404, None), self.check_cache(file, url=url), file) + def check_cache(self, file, url=None): """Helper to check if cache miss or hit""" url = urljoin(url or self.url, file) diff --git a/pulpcore/tests/unit/test_cache.py b/pulpcore/tests/unit/test_cache.py index 0257857222..93cce65ca3 100644 --- a/pulpcore/tests/unit/test_cache.py +++ b/pulpcore/tests/unit/test_cache.py @@ -33,6 +33,7 @@ def test_03_basic_delete(self): def test_04_basic_expires(self): """Tests setting values with expiration times""" + self.skipTest("Timing is inconsistent in CI") cache = Cache() cache.set("key", "hi", expires=5) ret = cache.get("key") From 1740c86e33eb0fc83cf16c6d509922cc4b9ed8e2 Mon Sep 17 00:00:00 2001 From: David Davis Date: Tue, 13 Jul 2021 14:58:56 -0400 Subject: [PATCH 18/79] Reduce the cache entry expiration TTL If Redis goes down then its persisted data can get out of sync with Pulp, because cache invalidation events cannot occur. We should mitigate the impact by letting cache entries expire after a shorter period of time. backports: #8996 https://pulp.plan.io/issues/8996 fixes #9063 (cherry picked from commit 017b0b1e97de655f79a0192aa5605c0d9163877c) --- CHANGES/9063.misc | 2 ++ docs/configuration/settings.rst | 2 ++ pulpcore/app/settings.py | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 CHANGES/9063.misc diff --git a/CHANGES/9063.misc b/CHANGES/9063.misc new file mode 100644 index 0000000000..2deb4e83ee --- /dev/null +++ b/CHANGES/9063.misc @@ -0,0 +1,2 @@ +Reduced the default expiration TTL of entries in the Pulp content app cache. +(backported from #8996) diff --git a/docs/configuration/settings.rst b/docs/configuration/settings.rst index 70655b498f..e9e1d06c96 100644 --- a/docs/configuration/settings.rst +++ b/docs/configuration/settings.rst @@ -211,6 +211,8 @@ CACHE_SETTINGS * ``EXPIRES_TTL`` - Number of seconds entries should stay in the cache before expiring. + Defaults to ``600`` seconds. + .. note:: Set to ``None`` to have entries not expire. Content app responses are always invalidated when the backing distribution is updated. diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py index e4527a6107..578007f788 100644 --- a/pulpcore/app/settings.py +++ b/pulpcore/app/settings.py @@ -242,7 +242,7 @@ # https://docs.pulpproject.org/pulpcore/configuration/settings.html#pulp-cache CACHE_ENABLED = True CACHE_SETTINGS = { - "EXPIRES_TTL": 86400, + "EXPIRES_TTL": 600, # 10 minutes } SPECTACULAR_SETTINGS = { From b09d2ae67aae5d5bc6ddd16984a5cb08ac3de16a Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 13 Jul 2021 19:48:33 +0000 Subject: [PATCH 19/79] Building changelog for 3.14.2 [noissue] --- CHANGES.rst | 39 +++++++++++++++++++++++++++++++++++++++ CHANGES/9057.bugfix | 4 ---- CHANGES/9058.bugfix | 2 -- CHANGES/9059.bugfix | 2 -- CHANGES/9063.misc | 2 -- CHANGES/9068.bugfix | 2 -- CHANGES/9077.bugfix | 2 -- 7 files changed, 39 insertions(+), 14 deletions(-) delete mode 100644 CHANGES/9057.bugfix delete mode 100644 CHANGES/9058.bugfix delete mode 100644 CHANGES/9059.bugfix delete mode 100644 CHANGES/9063.misc delete mode 100644 CHANGES/9068.bugfix delete mode 100644 CHANGES/9077.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index a6bb5101a4..b71af069e9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,45 @@ Changelog .. towncrier release notes start +3.14.2 (2021-07-13) +=================== +REST API +-------- + +Bugfixes +~~~~~~~~ + +- Fixed bug where content app would not respond to ``Range`` HTTP Header in requests when + ``remote.policy`` was either ``on_demand`` or ``streamed``. For example this request is used by + Anaconda clients. + (backported from #8865) + `#9057 `_ +- Fixed a bug that caused a serializer to ignore form data for ``pulp_labels``. + (backported from #8954) + `#9058 `_ +- Fixed the behavior of setting "repository" on a distribution for publication-based plugins. + (backported from #9039) + `#9059 `_ +- Use proxy auth from Remote config to download content from a remote repository. + (backported from #9024) + `#9068 `_ +- Fixed server error when accessing invalid files from content app base directory + (backported from #9074) + `#9077 `_ + + +Misc +~~~~ + +- `#9063 `_ + + +Plugin API +---------- + +No significant changes. + + 3.14.1 (2021-07-07) =================== REST API diff --git a/CHANGES/9057.bugfix b/CHANGES/9057.bugfix deleted file mode 100644 index fdb8dc2594..0000000000 --- a/CHANGES/9057.bugfix +++ /dev/null @@ -1,4 +0,0 @@ -Fixed bug where content app would not respond to ``Range`` HTTP Header in requests when -``remote.policy`` was either ``on_demand`` or ``streamed``. For example this request is used by -Anaconda clients. -(backported from #8865) diff --git a/CHANGES/9058.bugfix b/CHANGES/9058.bugfix deleted file mode 100644 index 3ffcee15f1..0000000000 --- a/CHANGES/9058.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a bug that caused a serializer to ignore form data for ``pulp_labels``. -(backported from #8954) diff --git a/CHANGES/9059.bugfix b/CHANGES/9059.bugfix deleted file mode 100644 index d551291da5..0000000000 --- a/CHANGES/9059.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed the behavior of setting "repository" on a distribution for publication-based plugins. -(backported from #9039) diff --git a/CHANGES/9063.misc b/CHANGES/9063.misc deleted file mode 100644 index 2deb4e83ee..0000000000 --- a/CHANGES/9063.misc +++ /dev/null @@ -1,2 +0,0 @@ -Reduced the default expiration TTL of entries in the Pulp content app cache. -(backported from #8996) diff --git a/CHANGES/9068.bugfix b/CHANGES/9068.bugfix deleted file mode 100644 index 6e9f2c31d3..0000000000 --- a/CHANGES/9068.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Use proxy auth from Remote config to download content from a remote repository. -(backported from #9024) diff --git a/CHANGES/9077.bugfix b/CHANGES/9077.bugfix deleted file mode 100644 index b98bba2737..0000000000 --- a/CHANGES/9077.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed server error when accessing invalid files from content app base directory -(backported from #9074) From 64bc5e5310fa9d628753efac6c69251270f6398c Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 13 Jul 2021 19:48:33 +0000 Subject: [PATCH 20/79] Release 3.14.2 Redmine Query: https://pulp.plan.io/issues?set_filter=1&status_id=*&issue_id=9077,9059,9068,9057,9058,9063 Redmine Milestone: https://pulp.plan.io/versions/259.json [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 01cc8a2661..efd1b740a9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.2.dev +current_version = 3.14.2 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 5b5e0b743c..2be31a7544 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.2.dev" +version = "3.14.2" # The full version, including alpha/beta/rc tags. -release = "3.14.2.dev" +release = "3.14.2" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index a5ee095eb5..1c57265347 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.2.dev" + version = "3.14.2" def ready(self): super().ready() diff --git a/setup.py b/setup.py index ba5a374fd5..8344e8335a 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.2.dev", + version="3.14.2", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From fbf9daf5e0c9b318ef87a18955df6482d1a119b7 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 13 Jul 2021 19:48:33 +0000 Subject: [PATCH 21/79] Bump to 3.14.3.dev [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index efd1b740a9..80d149a250 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.2 +current_version = 3.14.3.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 2be31a7544..840d51564c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.2" +version = "3.14.3.dev" # The full version, including alpha/beta/rc tags. -release = "3.14.2" +release = "3.14.3.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 1c57265347..c6181cd637 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.2" + version = "3.14.3.dev" def ready(self): super().ready() diff --git a/setup.py b/setup.py index 8344e8335a..64ec6f6b40 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.2", + version="3.14.3.dev", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 8ea4554af81316653358802152b116f174d54cf6 Mon Sep 17 00:00:00 2001 From: David Davis Date: Fri, 16 Jul 2021 09:20:27 -0400 Subject: [PATCH 22/79] Update CI files [noissue] --- .ci/ansible/Containerfile.j2 | 2 +- .github/workflows/scripts/before_install.sh | 8 +-- .../scripts/install_python_client.sh | 2 +- .github/workflows/scripts/script.sh | 2 +- .github/workflows/scripts/update_ci.sh | 25 ++++++++++ .github/workflows/update_ci.yml | 49 +++++++++++++++++++ template_config.yml | 2 + 7 files changed, 83 insertions(+), 7 deletions(-) create mode 100755 .github/workflows/scripts/update_ci.sh create mode 100644 .github/workflows/update_ci.yml diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 66c46cc40b..aa0f608b6a 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -23,7 +23,7 @@ RUN pip3 install \ RUN mkdir -p /etc/nginx/pulp/ {% for item in plugins %} -RUN ln /usr/local/lib/python3.6/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true +RUN ln /usr/local/lib/python3.8/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true {% endfor %} ENTRYPOINT ["/init"] diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 33d6b07b1f..4eb45526e9 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -27,6 +27,9 @@ else BRANCH="${GITHUB_REF##refs/tags/}" fi +COMMIT_MSG=$(git log --format=%B --no-merges -1) +export COMMIT_MSG + if [[ "$TEST" == "upgrade" ]]; then git checkout -b ci_upgrade_test cp -R .github /tmp/.github @@ -42,7 +45,7 @@ fi if [[ "$TEST" == "plugin-from-pypi" ]]; then COMPONENT_VERSION=$(http https://pypi.org/pypi/pulpcore/json | jq -r '.info.version') else - COMPONENT_VERSION=$(sed -ne "s/\s*version=['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) + COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) fi mkdir .ci/ansible/vars || true echo "---" > .ci/ansible/vars/main.yaml @@ -53,9 +56,6 @@ echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh -COMMIT_MSG=$(git log --format=%B --no-merges -1) -export COMMIT_MSG - if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index da768052ba..c31e7b8cae 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -41,6 +41,6 @@ rm -rf pulpcore-client ./generate.sh pulpcore python $VERSION cd pulpcore-client python setup.py sdist bdist_wheel --python-tag py3 -pip install dist/pulpcore_client-$VERSION-py3-none-any.whl +find . -name "*.whl" -exec pip install {} \; tar cvf ../../pulpcore/python-client.tar ./dist exit $? diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index c2ec615c25..6774945204 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -104,7 +104,7 @@ echo "Checking for uncommitted migrations..." cmd_prefix bash -c "django-admin makemigrations --check --dry-run" # Run unit tests. -cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.6/site-packages/pulpcore/tests/unit/" +cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.8/site-packages/pulpcore/tests/unit/" # Run functional tests export PYTHONPATH=$REPO_ROOT${PYTHONPATH:+:${PYTHONPATH}} diff --git a/.github/workflows/scripts/update_ci.sh b/.github/workflows/scripts/update_ci.sh new file mode 100755 index 0000000000..b7fc256310 --- /dev/null +++ b/.github/workflows/scripts/update_ci.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -eu + +if [ ! -d ../plugin_template ]; then + echo "Checking out plugin_template" + git clone https://github.com/pulp/plugin_template.git ../plugin_template +fi + + +if [ ! -f "template_config.yml" ]; then + echo "No template_config.yml detected." + exit 1 +fi + +pushd ../plugin_template +./plugin-template --github pulpcore +popd + +if [[ `git status --porcelain` ]]; then + git add -A + git commit -m "Update CI files" -m "[noissue]" +else + echo "No updates needed" +fi diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml new file mode 100644 index 0000000000..e3fbedc9e5 --- /dev/null +++ b/.github/workflows/update_ci.yml @@ -0,0 +1,49 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulpcore' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template +--- +name: CI Update +on: + schedule: + # * is a special character in YAML so you have to quote this string + # runs at 2:30 UTC daily + - cron: '30 2 * * *' + + workflow_dispatch: + +jobs: + update: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Configure Git with pulpbot name and email + run: | + git config --global user.name 'pulpbot' + git config --global user.email 'pulp-infra@redhat.com' + + - name: Run update + run: | + .github/workflows/scripts/update_ci.sh + + - name: Create Pull Request for CI files + uses: peter-evans/create-pull-request@v3 + with: + committer: pulpbot + author: pulpbot + branch: ${{ github.event.inputs.release }}-update-ci-files + base: ${{ github.event.inputs.release }} + title: 'Update CI for ${{ github.event.inputs.release }}' + body: '[noissue]' + commit-message: | + Update CI files for ${{ github.event.inputs.release }}' + + [noissue] + delete-branch: true diff --git a/template_config.yml b/template_config.yml index b64cba52b8..9c00aa869b 100644 --- a/template_config.yml +++ b/template_config.yml @@ -21,6 +21,7 @@ deploy_daily_client_to_rubygems: true deploy_to_pypi: true docker_fixtures: true docs_test: true +flake8: true issue_tracker: redmine plugin_app_label: core plugin_camel: Pulpcore @@ -47,6 +48,7 @@ pypi_username: pulp redmine_project: pulp release_user: pulpbot stable_branch: null +sync_ci: true test_bindings: true test_cli: true test_fips_nightly: false From ae451b817acc2df6a64ed09649606ef83242ae41 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Thu, 15 Jul 2021 10:07:06 +0200 Subject: [PATCH 23/79] Fixed non-gracefully shutting down workers When hitting ctrl-c twice, workers should also kill their supervised tasks. And shut them down. backports #8986 https://pulp.plan.io/issues/8986 fixes #9086 (cherry picked from commit 80d7d6d8e24c26d6eb0ff13ce29ed8c764cf2341) --- CHANGES/9086.bugfix | 2 ++ pulpcore/tasking/pulpcore_worker.py | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 CHANGES/9086.bugfix diff --git a/CHANGES/9086.bugfix b/CHANGES/9086.bugfix new file mode 100644 index 0000000000..383de68c0e --- /dev/null +++ b/CHANGES/9086.bugfix @@ -0,0 +1,2 @@ +Fixed signal handling to properly kill a task when double ctrl-c is used to shut down a worker fast. +(backported from #8986) diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py index 9818a454b5..4a4ef31151 100644 --- a/pulpcore/tasking/pulpcore_worker.py +++ b/pulpcore/tasking/pulpcore_worker.py @@ -234,6 +234,12 @@ def run_forever(self): def child_signal_handler(sig, frame): + # Reset signal handlers to default + # If you kill the process a second time it's not graceful anymore. + signal.signal(signal.SIGINT, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGUSR1, signal.SIG_DFL) + if sig == signal.SIGUSR1: sys.exit() From 851927fdd44e4b4a564484b030871c0bfeec8831 Mon Sep 17 00:00:00 2001 From: David Davis Date: Fri, 16 Jul 2021 13:53:36 -0400 Subject: [PATCH 24/79] Fix CLI test after config file name change [noissue] --- .github/workflows/scripts/before_install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 4eb45526e9..ca69ef6fc6 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -108,9 +108,9 @@ fi cd pulp-cli pip install -e . -pulp config create --base-url http://pulp --location tests/settings.toml --no-verify-ssl +pulp config create --base-url http://pulp --location tests/cli.toml --no-verify-ssl mkdir ~/.config/pulp -cp tests/settings.toml ~/.config/pulp/settings.toml +cp tests/cli.toml ~/.config/pulp/cli.toml cd .. From fc8d65688f4a423db4bbed8424d10a6a5ebcec6c Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Mon, 19 Jul 2021 09:04:25 +0200 Subject: [PATCH 25/79] Validate absolute pathnames in remotes' URLs Before this change, it was not possible to determine why did the synchronization fail when a user provided a seemingly valid URL. This commit also adds more relevant information to the error message. Having set `ALLOWED_EXPORT_PATHS` to `["/tmp", "/home/vagrant/test"]`, the following error messages are shown: ``` $ pulp file remote create --name test --url file://error/vagrant/test/centos-7/PULP_MANIFEST Error: {"url":["The path 'error/vagrant/test/centos-7/PULP_MANIFEST' needs to be an absolute pathname."]} $ pulp file remote create --name test --url file:///error/vagrant/test/centos-7/PULP_MANIFEST Error: {"url":["The path '/error/vagrant/test/centos-7/PULP_MANIFEST' does not start with any of the allowed import paths"]} ``` backports #9080 fixes #9083 (cherry picked from commit d8cba5abdddbe2297e4b7c72571f08c55d88bcd4) --- CHANGES/9083.bugfix | 2 + pulpcore/app/serializers/repository.py | 24 ++++++---- .../api/using_plugin/test_crud_repos.py | 47 +++++++++++++++++++ 3 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 CHANGES/9083.bugfix diff --git a/CHANGES/9083.bugfix b/CHANGES/9083.bugfix new file mode 100644 index 0000000000..5facfc4849 --- /dev/null +++ b/CHANGES/9083.bugfix @@ -0,0 +1,2 @@ +Fixed improper validation of remotes' URLs. +(backported from #9080) diff --git a/pulpcore/app/serializers/repository.py b/pulpcore/app/serializers/repository.py index df187e5069..0f55ebad3b 100644 --- a/pulpcore/app/serializers/repository.py +++ b/pulpcore/app/serializers/repository.py @@ -205,14 +205,14 @@ class RemoteSerializer(ModelSerializer): required=False, ) - def validate_url(self, value): + def validate_url(self, url): """ Check if the 'url' is a ``file://`` path, and if so, ensure it's an ALLOWED_IMPORT_PATH. The ALLOWED_IMPORT_PATH is specified as a Pulp setting. Args: - value: The user-provided value for 'url' to be validated. + url: The user-provided value for 'url' to be validated. Raises: ValidationError: When the url starts with `file://`, but is not a subfolder of a path in @@ -221,16 +221,24 @@ def validate_url(self, value): Returns: The validated value. """ - if not value.lower().startswith("file://"): - return value + if not url.lower().startswith("file://"): + return url - user_path = value[7:] + user_path = url[7:] + if not os.path.isabs(user_path): + raise serializers.ValidationError( + _("The path '{}' needs to be an absolute pathname.").format(user_path) + ) + + user_provided_realpath = os.path.realpath(user_path) for allowed_path in settings.ALLOWED_IMPORT_PATHS: - user_provided_realpath = os.path.realpath(user_path) if user_provided_realpath.startswith(allowed_path): - return value - raise serializers.ValidationError(_("url '{}' is not an allowed import path").format(value)) + return url + + raise serializers.ValidationError( + _("The path '{}' does not start with any of the allowed import paths").format(user_path) + ) def validate_proxy_url(self, value): """ diff --git a/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py b/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py index 654826cf8e..7d4f52c658 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py +++ b/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py @@ -357,3 +357,50 @@ def test_headers(self): self.remotes_api.partial_update(self.remote.pulp_href, data) data = {"headers": [{"Connection": "keep-alive"}]} self.remotes_api.partial_update(self.remote.pulp_href, data) + + +class RemoteFileURLsValidationTestCase(unittest.TestCase): + """A test case that verifies the validation of remotes' URLs.""" + + @classmethod + def setUpClass(cls): + """Initialize class-wide variables""" + cls.cfg = config.get_config() + + cls.api_client = api.Client(cls.cfg, api.json_handler) + cls.file_client = FileApiClient(cls.cfg.get_bindings_config()) + cls.remotes_api = RemotesFileApi(cls.file_client) + + def test_invalid_absolute_pathname(self): + """Test the validation of an invalid absolute pathname.""" + remote_attrs = { + "name": utils.uuid4(), + "url": "file://error/path/name", + } + self.raise_for_invalid_request(remote_attrs) + + def test_invalid_import_path(self): + """Test the validation of an invalid import pathname.""" + remote_attrs = { + "name": utils.uuid4(), + "url": "file:///error/path/name", + } + self.raise_for_invalid_request(remote_attrs) + + def raise_for_invalid_request(self, remote_attrs): + """Check if Pulp returns HTTP 400 after issuing an invalid request.""" + with self.assertRaises(ApiException) as ae: + remote = self.remotes_api.create(remote_attrs) + self.addCleanup(self.remotes_api.delete, remote.pulp_href) + + self.assertEqual(ae.exception.status, 400) + + def test_valid_import_path(self): + """Test the creation of a remote after passing a valid URL.""" + remote_attrs = { + "name": utils.uuid4(), + "url": "file:///tmp/good", + } + + remote = self.remotes_api.create(remote_attrs) + self.addCleanup(self.remotes_api.delete, remote.pulp_href) From dd4299afa0fb7d9de055a81857a09d2adeb07817 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Mon, 19 Jul 2021 09:05:44 +0200 Subject: [PATCH 26/79] Conditionally include Redis connection information Redis is only used when cache is enabled or RQ tasking is used. In other cases, the connection is not needed. We've changed the Foreman installer to only install Redis if needed, just to stop wasting resources[1]. However, this leads to warnings. This PR changes the status view to only include Redis if it's used. [1]: https://github.com/theforeman/puppet-pulpcore/commit/7a5543b1a47de8d76a1cad1fb76e12db7bbf8e45 backports #9070 https://pulp.plan.io/issues/9070 fixes #9085 (cherry picked from commit 8ab8afb45675cc9c420eb1872c0dfcfd973d657b) --- CHANGES/9085.bugfix | 3 +++ pulpcore/app/serializers/status.py | 5 ++++- pulpcore/app/views/status.py | 6 +++++- pulpcore/tests/functional/api/test_status.py | 11 +++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 CHANGES/9085.bugfix diff --git a/CHANGES/9085.bugfix b/CHANGES/9085.bugfix new file mode 100644 index 0000000000..1616c9cfe3 --- /dev/null +++ b/CHANGES/9085.bugfix @@ -0,0 +1,3 @@ +Set Redis connection information in status to null unless it's used. Redis is +needed for RQ tasking or content caching. +(backported from #9070) diff --git a/pulpcore/app/serializers/status.py b/pulpcore/app/serializers/status.py index 79cf4f0f22..abd48d643d 100644 --- a/pulpcore/app/serializers/status.py +++ b/pulpcore/app/serializers/status.py @@ -74,6 +74,9 @@ class StatusSerializer(serializers.Serializer): help_text=_("Database connection information") ) - redis_connection = RedisConnectionSerializer(help_text=_("Redis connection information")) + redis_connection = RedisConnectionSerializer( + required=False, + help_text=_("Redis connection information"), + ) storage = StorageSerializer(required=False, help_text=_("Storage information")) diff --git a/pulpcore/app/views/status.py b/pulpcore/app/views/status.py index 55a74a91a3..c9d035652d 100644 --- a/pulpcore/app/views/status.py +++ b/pulpcore/app/views/status.py @@ -57,7 +57,11 @@ def get(self, request): for app in pulp_plugin_configs(): versions.append({"component": app.label, "version": app.version}) - redis_status = {"connected": self._get_redis_conn_status()} + if settings.CACHE_ENABLED or not settings.USE_NEW_WORKER_TYPE: + redis_status = {"connected": self._get_redis_conn_status()} + else: + redis_status = None + db_status = {"connected": self._get_db_conn_status()} try: diff --git a/pulpcore/tests/functional/api/test_status.py b/pulpcore/tests/functional/api/test_status.py index 169d51293c..8ff15dc98f 100644 --- a/pulpcore/tests/functional/api/test_status.py +++ b/pulpcore/tests/functional/api/test_status.py @@ -1,6 +1,7 @@ """Test the status page.""" import unittest +from django.test import override_settings from jsonschema import validate from pulp_smash import api, cli, config, utils from pulp_smash.pulp3.constants import STATUS_PATH @@ -91,6 +92,7 @@ def verify_get_response(self, status): """ validate(status, self.status_response) self.assertTrue(status["database_connection"]["connected"]) + self.assertIsNotNone(status["redis_connection"]) self.assertTrue(status["redis_connection"]["connected"]) self.assertNotEqual(status["online_workers"], []) self.assertNotEqual(status["versions"], []) @@ -98,3 +100,12 @@ def verify_get_response(self, status): self.assertIsNotNone(status["storage"]) else: self.assertIsNone(status["storage"]) + + @override_settings(CACHE_ENABLED=False, USE_NEW_WORKER_TYPE=True) + def verify_get_response_without_redis(self, status): + """Verify the response to an HTTP GET call when Redis is not used. + + Verify that redis_connection is null + """ + validate(status, self.status_response) + self.assertIsNone(status["redis_connection"]) From 0e562768c7dca4ea0f62c860dc3e75003235dcea Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Thu, 22 Jul 2021 16:23:07 +0200 Subject: [PATCH 27/79] Prevent tasks being assigned to missing workers backports #8779 fixes #9116 (cherry picked from commit 0cfaa8e7433b7cd272631a6f51b9f4a7b10224a7) --- CHANGES/9116.bugfix | 2 ++ pulpcore/tasking/tasks.py | 8 +++++++- pulpcore/tasking/worker.py | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 CHANGES/9116.bugfix diff --git a/CHANGES/9116.bugfix b/CHANGES/9116.bugfix new file mode 100644 index 0000000000..cc5be00c01 --- /dev/null +++ b/CHANGES/9116.bugfix @@ -0,0 +1,2 @@ +Fixed a bug, where new tasks were assigned to dead workers. +(backported from #8779) diff --git a/pulpcore/tasking/tasks.py b/pulpcore/tasking/tasks.py index ba74ab7ca7..424bd37ec8 100644 --- a/pulpcore/tasking/tasks.py +++ b/pulpcore/tasking/tasks.py @@ -47,15 +47,19 @@ def _acquire_worker(resources): """ # Find a worker who already has one of the reservations, it is safe to send this work to them try: - return ( + worker = ( Worker.objects.select_for_update() .filter(pk__in=Worker.objects.filter(reservations__resource__in=resources)) .get() ) + if worker.online: + return worker except Worker.MultipleObjectsReturned: raise Worker.DoesNotExist except Worker.DoesNotExist: pass + else: # no Exception raised -> worker is offline; wait for it to be cleaned up + raise Worker.DoesNotExist # Otherwise, return any available worker at random workers_qs = Worker.objects.online_workers().exclude( @@ -146,6 +150,8 @@ def _queue_reserved_task(func, inner_task_id, resources, inner_args, inner_kwarg worker=worker, resource=resource ) TaskReservedResource.objects.create(resource=reservation, task=task_status) + worker.cleaned_up = False + worker.save(update_fields=["cleaned_up"]) except (Worker.DoesNotExist, IntegrityError): # if worker is ready, or we have a worker but we can't create the reservations, wait time.sleep(0.25) diff --git a/pulpcore/tasking/worker.py b/pulpcore/tasking/worker.py index 972760774e..eecfb3d0ad 100644 --- a/pulpcore/tasking/worker.py +++ b/pulpcore/tasking/worker.py @@ -22,6 +22,7 @@ from pulpcore.app.settings import WORKER_TTL # noqa: E402: module level not at top of file from pulpcore.app.models import Task # noqa: E402: module level not at top of file +from pulpcore.constants import TASK_STATES # noqa: E402: module level not at top of file from pulpcore.tasking.constants import ( # noqa: E402: module level not at top of file TASKING_CONSTANTS, @@ -105,6 +106,8 @@ def perform_job(self, job, queue): except Task.DoesNotExist: pass else: + if task.state != TASK_STATES.WAITING: + return task.set_running() user = get_users_with_perms(task).first() _set_current_user(user) From 87039f779df198331e007a6c0526d75b15c3d964 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Fri, 23 Jul 2021 11:47:47 -0400 Subject: [PATCH 28/79] Change content app's working directory dynamically As of this commit, content app is no longer storing temporary files in the /var/run/ directory. The temporary files were created during on-demand downloading and were not removed until, e.g., restarting pulp services. backports #9000 fixes #9110 (cherry picked from commit f8bd7f8a2e696d1f429b3aae2886afbcd66bc504) --- CHANGES/9110.bugfix | 2 ++ pulpcore/content/__init__.py | 2 ++ pulpcore/content/handler.py | 6 ++++++ pulpcore/tests/unit/content/test_handler.py | 14 ++++++++------ 4 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 CHANGES/9110.bugfix diff --git a/CHANGES/9110.bugfix b/CHANGES/9110.bugfix new file mode 100644 index 0000000000..c199885a9b --- /dev/null +++ b/CHANGES/9110.bugfix @@ -0,0 +1,2 @@ +Fixed a bug where on-demand downloads would fill up ``/var/run/`` by not deleting downloaded files. +(backported from #9000) diff --git a/pulpcore/content/__init__.py b/pulpcore/content/__init__.py index 98bb8e5735..3e29f477f5 100644 --- a/pulpcore/content/__init__.py +++ b/pulpcore/content/__init__.py @@ -55,6 +55,8 @@ def save_heartbeat_blocking(): async def server(*args, **kwargs): + os.chdir(settings.WORKING_DIRECTORY) + asyncio.ensure_future(_heartbeat()) for pulp_plugin in pulp_plugin_configs(): if pulp_plugin.name != "pulpcore.app": diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py index 2f896ca6a7..eba1c27040 100644 --- a/pulpcore/content/handler.py +++ b/pulpcore/content/handler.py @@ -720,6 +720,12 @@ def _save_artifact(self, download_result, remote_artifact): **download_result.artifact_attributes, file=download_result.path ) artifact.save() + else: + # The file needs to be unlinked because it was not used to create an artifact. + # The artifact must have already been saved while servicing another request for + # the same artifact. + os.unlink(download_result.path) + update_content_artifact = True if content_artifact._state.adding: # This is the first time pull-through content was requested. diff --git a/pulpcore/tests/unit/content/test_handler.py b/pulpcore/tests/unit/content/test_handler.py index 937d2de69b..b8a27cc949 100644 --- a/pulpcore/tests/unit/content/test_handler.py +++ b/pulpcore/tests/unit/content/test_handler.py @@ -1,6 +1,7 @@ +import tempfile + from unittest.mock import Mock -from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase from pulpcore.content import Handler @@ -20,18 +21,19 @@ def setUp(self): ) self.ra2 = Mock(content_artifact=self.ca2) - def download_result_mock(self, path): + def download_result_mock(self): dr = Mock() dr.artifact_attributes = {"size": 0} for digest_type in Artifact.DIGEST_FIELDS: dr.artifact_attributes[digest_type] = "abc123" - dr.path = SimpleUploadedFile(name=path, content="") + tmp_file = tempfile.NamedTemporaryFile(delete=False) + dr.path = tmp_file.name return dr def test_save_artifact(self): """Artifact needs to be created.""" cch = Handler() - new_artifact = cch._save_artifact(self.download_result_mock("c1"), self.ra1) + new_artifact = cch._save_artifact(self.download_result_mock(), self.ra1) c1 = Content.objects.get(pk=self.c1.pk) self.assertIsNotNone(new_artifact) self.assertEqual(c1._artifacts.get().pk, new_artifact.pk) @@ -39,9 +41,9 @@ def test_save_artifact(self): def test_save_artifact_artifact_already_exists(self): """Artifact turns out to already exist.""" cch = Handler() - new_artifact = cch._save_artifact(self.download_result_mock("c1"), self.ra1) + new_artifact = cch._save_artifact(self.download_result_mock(), self.ra1) - existing_artifact = cch._save_artifact(self.download_result_mock("c2"), self.ra2) + existing_artifact = cch._save_artifact(self.download_result_mock(), self.ra2) c2 = Content.objects.get(pk=self.c2.pk) self.assertEqual(existing_artifact.pk, new_artifact.pk) self.assertEqual(c2._artifacts.get().pk, existing_artifact.pk) From b4a373064231c2df605f37c2dd35b222520b0014 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Fri, 23 Jul 2021 11:48:34 -0400 Subject: [PATCH 29/79] Move downloaded files instead of copying them Before this commit, the condition that checks whether pulp's temporary files are stored within /var/lib/pulp was never met because it looked for files starting with /var/lib/pulp/media (MEDIA_ROOT). MEDIA_ROOT is however no longer pointing to /var/lib/pulp. backports #8295 fixes #9103 (cherry picked from commit fbe2d7f9c85008e8223fec8fcfdb87e665e06881) --- CHANGES/9103.bugfix | 2 ++ pulpcore/app/models/storage.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 CHANGES/9103.bugfix diff --git a/CHANGES/9103.bugfix b/CHANGES/9103.bugfix new file mode 100644 index 0000000000..43da1f6fcb --- /dev/null +++ b/CHANGES/9103.bugfix @@ -0,0 +1,2 @@ +Improved disk usage during the synchronization. +(backported from #8295) diff --git a/pulpcore/app/models/storage.py b/pulpcore/app/models/storage.py index 697fc7f211..d558e82125 100644 --- a/pulpcore/app/models/storage.py +++ b/pulpcore/app/models/storage.py @@ -11,9 +11,9 @@ class FileSystem(FileSystemStorage): """ Django's FileSystemStorage with modified _save() and get_available_name behaviors - The _save() will check if the file is saved in MEDIA_ROOT first. If it is, a move is used. This + The _save() will check if the file is saved in DEPLOY_ROOT first. If it is, a move is used. This will move all files created by the Downloaders and uploaded files from the user. If it is saved - in-memory or outside of MEDIA_ROOT, the data is written in chunks to the new location. + in-memory or outside of DEPLOY_ROOT, the data is written/copied in chunks to the new location. """ def get_available_name(self, name, max_length=None): @@ -63,7 +63,7 @@ def _save(self, name, content, max_length=None): try: if hasattr(content, "temporary_file_path") and content.temporary_file_path().startswith( - settings.MEDIA_ROOT + str(settings.DEPLOY_ROOT) ): file_move_safe(content.temporary_file_path(), full_path) else: From 99e335a23a22a228dcd5d41eb34d19c49c5e1c23 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Fri, 23 Jul 2021 16:39:31 +0000 Subject: [PATCH 30/79] Building changelog for 3.14.3 [noissue] --- CHANGES.rst | 35 +++++++++++++++++++++++++++++++++++ CHANGES/9083.bugfix | 2 -- CHANGES/9085.bugfix | 3 --- CHANGES/9086.bugfix | 2 -- CHANGES/9103.bugfix | 2 -- CHANGES/9110.bugfix | 2 -- CHANGES/9116.bugfix | 2 -- 7 files changed, 35 insertions(+), 13 deletions(-) delete mode 100644 CHANGES/9083.bugfix delete mode 100644 CHANGES/9085.bugfix delete mode 100644 CHANGES/9086.bugfix delete mode 100644 CHANGES/9103.bugfix delete mode 100644 CHANGES/9110.bugfix delete mode 100644 CHANGES/9116.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index b71af069e9..6bd02cb3fb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,41 @@ Changelog .. towncrier release notes start +3.14.3 (2021-07-23) +=================== +REST API +-------- + +Bugfixes +~~~~~~~~ + +- Fixed improper validation of remotes' URLs. + (backported from #9080) + `#9083 `_ +- Set Redis connection information in status to null unless it's used. Redis is + needed for RQ tasking or content caching. + (backported from #9070) + `#9085 `_ +- Fixed signal handling to properly kill a task when double ctrl-c is used to shut down a worker fast. + (backported from #8986) + `#9086 `_ +- Improved disk usage during the synchronization. + (backported from #8295) + `#9103 `_ +- Fixed a bug where on-demand downloads would fill up ``/var/run/`` by not deleting downloaded files. + (backported from #9000) + `#9110 `_ +- Fixed a bug, where new tasks were assigned to dead workers. + (backported from #8779) + `#9116 `_ + + +Plugin API +---------- + +No significant changes. + + 3.14.2 (2021-07-13) =================== REST API diff --git a/CHANGES/9083.bugfix b/CHANGES/9083.bugfix deleted file mode 100644 index 5facfc4849..0000000000 --- a/CHANGES/9083.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed improper validation of remotes' URLs. -(backported from #9080) diff --git a/CHANGES/9085.bugfix b/CHANGES/9085.bugfix deleted file mode 100644 index 1616c9cfe3..0000000000 --- a/CHANGES/9085.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Set Redis connection information in status to null unless it's used. Redis is -needed for RQ tasking or content caching. -(backported from #9070) diff --git a/CHANGES/9086.bugfix b/CHANGES/9086.bugfix deleted file mode 100644 index 383de68c0e..0000000000 --- a/CHANGES/9086.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed signal handling to properly kill a task when double ctrl-c is used to shut down a worker fast. -(backported from #8986) diff --git a/CHANGES/9103.bugfix b/CHANGES/9103.bugfix deleted file mode 100644 index 43da1f6fcb..0000000000 --- a/CHANGES/9103.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Improved disk usage during the synchronization. -(backported from #8295) diff --git a/CHANGES/9110.bugfix b/CHANGES/9110.bugfix deleted file mode 100644 index c199885a9b..0000000000 --- a/CHANGES/9110.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a bug where on-demand downloads would fill up ``/var/run/`` by not deleting downloaded files. -(backported from #9000) diff --git a/CHANGES/9116.bugfix b/CHANGES/9116.bugfix deleted file mode 100644 index cc5be00c01..0000000000 --- a/CHANGES/9116.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a bug, where new tasks were assigned to dead workers. -(backported from #8779) From 0dc02e9a4a4241f593be52b804e5cdcd77d73b04 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Fri, 23 Jul 2021 16:39:31 +0000 Subject: [PATCH 31/79] Release 3.14.3 Redmine Query: https://pulp.plan.io/issues?set_filter=1&status_id=*&issue_id=9086,9116,9085,9110,9103,9083 Redmine Milestone: https://pulp.plan.io/versions/262.json [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 80d149a250..0f498f71c0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.3.dev +current_version = 3.14.3 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 840d51564c..fe535c135b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.3.dev" +version = "3.14.3" # The full version, including alpha/beta/rc tags. -release = "3.14.3.dev" +release = "3.14.3" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index c6181cd637..033d077338 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.3.dev" + version = "3.14.3" def ready(self): super().ready() diff --git a/setup.py b/setup.py index 64ec6f6b40..31549c964b 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.3.dev", + version="3.14.3", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 845a12a95ea7973855e3d3da74e6454e44883378 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Fri, 23 Jul 2021 16:39:32 +0000 Subject: [PATCH 32/79] Bump to 3.14.4.dev [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0f498f71c0..f7d952c435 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.3 +current_version = 3.14.4.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index fe535c135b..cc1ca31d26 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.3" +version = "3.14.4.dev" # The full version, including alpha/beta/rc tags. -release = "3.14.3" +release = "3.14.4.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 033d077338..927fc4a11a 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.3" + version = "3.14.4.dev" def ready(self): super().ready() diff --git a/setup.py b/setup.py index 31549c964b..b36420a758 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.3", + version="3.14.4.dev", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From cb1ad9756703beef704b14d08985a73a857687ac Mon Sep 17 00:00:00 2001 From: David Davis Date: Fri, 23 Jul 2021 15:25:02 -0400 Subject: [PATCH 33/79] Update CI files and set python_version to 3.6 [noissue] --- .ci/ansible/Containerfile.j2 | 3 ++- .github/template_gitref | 1 + .github/workflows/ci.yml | 6 +++--- .github/workflows/create-branch.yml | 2 +- .github/workflows/nightly.yml | 4 ++-- .github/workflows/release.yml | 6 +++--- .github/workflows/scripts/before_install.sh | 4 ++-- .github/workflows/scripts/install.sh | 8 +++++--- .github/workflows/scripts/script.sh | 2 +- .github/workflows/update_ci.yml | 8 ++++---- template_config.yml | 7 +++++-- 11 files changed, 29 insertions(+), 22 deletions(-) create mode 100644 .github/template_gitref diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index aa0f608b6a..de47e4dac8 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -23,7 +23,8 @@ RUN pip3 install \ RUN mkdir -p /etc/nginx/pulp/ {% for item in plugins %} -RUN ln /usr/local/lib/python3.8/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true +RUN export plugin_path="$(pip show {{ item.name }} | sed -n -e "s/Location: //p")/{{ item.name }}" +RUN ln "$plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf" || true {% endfor %} ENTRYPOINT ["/init"] diff --git a/.github/template_gitref b/.github/template_gitref new file mode 100644 index 0000000000..4bfaa589fc --- /dev/null +++ b/.github/template_gitref @@ -0,0 +1 @@ +2021.04.08-84-g16a91e9-dirty diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccd30c242e..46d42ff4a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" # dev_requirements contains tools needed for flake8, etc. - name: Install requirements @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - uses: actions/setup-ruby@v1 with: ruby-version: "2.6" @@ -156,7 +156,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - name: Install httpie run: | diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index a1dae0bcdd..8b2a7c772d 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - name: Install python dependencies run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 933b274a6b..b3de58e0b6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -34,7 +34,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - name: Install httpie run: | @@ -129,7 +129,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - uses: actions/setup-ruby@v1 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cec4dfe789..fe2c8a215f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - name: Install python dependencies run: | @@ -87,7 +87,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - uses: actions/setup-ruby@v1 with: ruby-version: "2.6" @@ -201,7 +201,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - uses: actions/setup-ruby@v1 with: diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index ca69ef6fc6..1fd8b8986c 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -116,7 +116,7 @@ cd .. -git clone --depth=1 https://github.com/pulp/pulp_file.git --branch master +git clone --depth=1 https://github.com/pulp/pulp_file.git --branch 1.8 if [ -n "$PULP_FILE_PR_NUMBER" ]; then cd pulp_file git fetch --depth=1 origin pull/$PULP_FILE_PR_NUMBER/head:$PULP_FILE_PR_NUMBER @@ -124,7 +124,7 @@ if [ -n "$PULP_FILE_PR_NUMBER" ]; then cd .. fi -git clone --depth=1 https://github.com/pulp/pulp-certguard.git --branch master +git clone --depth=1 https://github.com/pulp/pulp-certguard.git --branch 1.4 if [ -n "$PULP_CERTGUARD_PR_NUMBER" ]; then cd pulp-certguard git fetch --depth=1 origin pull/$PULP_CERTGUARD_PR_NUMBER/head:$PULP_CERTGUARD_PR_NUMBER diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 4053cfe2df..e0ecc2c392 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -29,13 +29,13 @@ TAG=ci_build if [ -e $REPO_ROOT/../pulp_file ]; then PULP_FILE=./pulp_file else - PULP_FILE=git+https://github.com/pulp/pulp_file.git@master + PULP_FILE=git+https://github.com/pulp/pulp_file.git@1.8 fi if [ -e $REPO_ROOT/../pulp-certguard ]; then PULP_CERTGUARD=./pulp-certguard else - PULP_CERTGUARD=git+https://github.com/pulp/pulp-certguard.git@master + PULP_CERTGUARD=git+https://github.com/pulp/pulp-certguard.git@1.4 fi if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then PLUGIN_NAME=./pulpcore/dist/pulpcore-$PLUGIN_VERSION-py3-none-any.whl @@ -63,7 +63,9 @@ VARSYAML cat >> vars/main.yaml << VARSYAML pulp_settings: {"allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"]} pulp_scheme: http -pulp_container_tag: latest + +pulp_container_tag: python36 + VARSYAML if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "upgrade" || "$TEST" == "s3" || "$TEST" == "plugin-from-pypi" ]]; then diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 6774945204..c2ec615c25 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -104,7 +104,7 @@ echo "Checking for uncommitted migrations..." cmd_prefix bash -c "django-admin makemigrations --check --dry-run" # Run unit tests. -cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.8/site-packages/pulpcore/tests/unit/" +cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.6/site-packages/pulpcore/tests/unit/" # Run functional tests export PYTHONPATH=$REPO_ROOT${PYTHONPATH:+:${PYTHONPATH}} diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index e3fbedc9e5..e0342596bd 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -38,12 +38,12 @@ jobs: with: committer: pulpbot author: pulpbot - branch: ${{ github.event.inputs.release }}-update-ci-files - base: ${{ github.event.inputs.release }} - title: 'Update CI for ${{ github.event.inputs.release }}' + branch: ${GITHUB_REF#refs/heads/}-update-ci-files + base: ${GITHUB_REF#refs/heads/} + title: 'Update CI for ${GITHUB_REF#refs/heads/}' body: '[noissue]' commit-message: | - Update CI files for ${{ github.event.inputs.release }}' + Update CI files for ${GITHUB_REF#refs/heads/} [noissue] delete-branch: true diff --git a/template_config.yml b/template_config.yml index 9c00aa869b..5ae3356cf2 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,10 +1,12 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. +# generated with plugin_template@2021.04.08-84-g3271e88 + additional_repos: -- branch: master +- branch: 1.8 name: pulp_file -- branch: master +- branch: 1.4 name: pulp-certguard black: true check_commit_message: true @@ -45,6 +47,7 @@ pulpcore_pip_version_specifier: null pulpprojectdotorg_key_id: d1a778bda808 pydocstyle: true pypi_username: pulp +python_version: '3.6' redmine_project: pulp release_user: pulpbot stable_branch: null From c185be3a3c89616d723a9a43dd7d3a7b6ad67021 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Fri, 30 Jul 2021 16:31:56 -0400 Subject: [PATCH 34/79] Update aioredis dependency to 2.0.0 (from 2.0.0a1) [noissue] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4423cfd6de..7bb30f4a62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ asyncio-throttle~=1.0 aiohttp~=3.7.4 aiodns~=3.0.0 aiofiles==0.7.0 -aioredis~=2.0.0a1 +aioredis~=2.0.0 backoff~=1.10.0 click<9.0 Django~=2.2.24 # LTS version, switch only if we have a compelling reason to From c8769d6fb23af23c7cd0acd6f288afcfbe14fcda Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Mon, 9 Aug 2021 14:44:03 -0400 Subject: [PATCH 35/79] Prevent access to unpublished content from content app backports: #8870 https://pulp.plan.io/issues/8870 Required PR: https://github.com/pulp/pulp_file/pull/547 fixes #9126 (cherry picked from commit 27a0dc7424769b2d5d7e98ebc4cf9cdba6cdc149) --- CHANGES/9126.bugfix | 3 ++ CHANGES/plugin_api/9126.feature | 3 ++ pulpcore/app/models/publication.py | 3 ++ pulpcore/content/handler.py | 2 +- .../api/using_plugin/test_distributions.py | 46 +++++++------------ 5 files changed, 27 insertions(+), 30 deletions(-) create mode 100644 CHANGES/9126.bugfix create mode 100644 CHANGES/plugin_api/9126.feature diff --git a/CHANGES/9126.bugfix b/CHANGES/9126.bugfix new file mode 100644 index 0000000000..a16d959059 --- /dev/null +++ b/CHANGES/9126.bugfix @@ -0,0 +1,3 @@ +Unpublished content can no longer be accessed from content app if publication based-plugin has +defined their distributions as publication serving +(backported from #8870) diff --git a/CHANGES/plugin_api/9126.feature b/CHANGES/plugin_api/9126.feature new file mode 100644 index 0000000000..b556177acf --- /dev/null +++ b/CHANGES/plugin_api/9126.feature @@ -0,0 +1,3 @@ +Distribution model has a new boolean class variable ``SERVE_FROM_PUBLICATION`` for plugins to declare +whether their distributions serve from publications or directly from repository versions +(backported from #8870) diff --git a/pulpcore/app/models/publication.py b/pulpcore/app/models/publication.py index 28a8c3ad4d..e3d8ac6f2e 100644 --- a/pulpcore/app/models/publication.py +++ b/pulpcore/app/models/publication.py @@ -381,6 +381,9 @@ class Distribution(MasterModel): repository_version (models.ForeignKey): RepositoryVersion to be served. """ + # If distribution serves publications, set by subclasses for proper handling in content app + SERVE_FROM_PUBLICATION = False + name = models.TextField(db_index=True, unique=True) base_path = models.TextField(unique=True) diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py index eba1c27040..03631328b2 100644 --- a/pulpcore/content/handler.py +++ b/pulpcore/content/handler.py @@ -557,7 +557,7 @@ def get_contentartifact_blocking(): request, StreamResponse(headers=headers), ca ) - if repo_version: + if repo_version and not publication and not distro.SERVE_FROM_PUBLICATION: if rel_path == "" or rel_path[-1] == "/": index_path = "{}index.html".format(rel_path) diff --git a/pulpcore/tests/functional/api/using_plugin/test_distributions.py b/pulpcore/tests/functional/api/using_plugin/test_distributions.py index 10210ee0b3..091124d2ec 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_distributions.py +++ b/pulpcore/tests/functional/api/using_plugin/test_distributions.py @@ -299,26 +299,12 @@ def setUp(self): def test_nonpublished_content_not_served(self): """Verify content that hasn't been published is not served.""" - # Create a repository - self.repo = self.repo_api.create(gen_repo()) - self.addCleanup(self.repo_api.delete, self.repo.pulp_href) - - # Create a remote - self.remote = self.remote_api.create(gen_file_remote()) - self.addCleanup(self.remote_api.delete, self.remote.pulp_href) - - # Sync the repository. - repository_sync_data = RepositorySyncURL(remote=self.remote.pulp_href) - sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data) - monitor_task(sync_response.task) - - # Create a distribution. - response = self.distributions_api.create( - {"name": "foo", "base_path": "bar/foo", "repository": self.repo.pulp_href} - ) - distribution_href = monitor_task(response.task).created_resources[0] - self.distribution = self.distributions_api.read(distribution_href) - self.addCleanup(self.distributions_api.delete, self.distribution.pulp_href) + self.setup_download_test("immediate", publish=False) + files = ["", "1.iso", "2.iso", "3.iso"] + for file in files: + with self.assertRaises(HTTPError, msg=f"{file}") as cm: + download_content_unit(self.cfg, self.distribution.to_dict(), file) + self.assertEqual(cm.exception.response.status_code, 404, f"{file}") def test_content_served_on_demand(self): """Assert that on_demand content can be properly downloaded.""" @@ -426,7 +412,7 @@ def test_content_served_immediate_with_range_request_start_value_larger_than_con headers={"Range": "bytes=10485860-10485870"}, ) - def setup_download_test(self, policy, url=None): + def setup_download_test(self, policy, url=None, publish=True): # Create a repository self.repo = self.repo_api.create(gen_repo()) self.addCleanup(self.repo_api.delete, self.repo.pulp_href) @@ -444,16 +430,18 @@ def setup_download_test(self, policy, url=None): sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) - # Create a publication. - publish_data = FileFilePublication(repository=self.repo.pulp_href) - publish_response = self.publications_api.create(publish_data) - publication_href = monitor_task(publish_response.task).created_resources[0] - self.addCleanup(self.publications_api.delete, publication_href) + if publish: + # Create a publication. + publish_data = FileFilePublication(repository=self.repo.pulp_href) + publish_response = self.publications_api.create(publish_data) + publication_href = monitor_task(publish_response.task).created_resources[0] + self.addCleanup(self.publications_api.delete, publication_href) + serve, served_href = "publication", publication_href + else: + serve, served_href = "repository", self.repo.pulp_href # Create a distribution. - response = self.distributions_api.create( - {"name": "foo", "base_path": "bar/foo", "publication": publication_href} - ) + response = self.distributions_api.create(gen_distribution(**{serve: served_href})) distribution_href = monitor_task(response.task).created_resources[0] self.distribution = self.distributions_api.read(distribution_href) self.addCleanup(self.distributions_api.delete, self.distribution.pulp_href) From d44ea6d767c07196fdbe5a7dc1749d9a30852c99 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Mon, 9 Aug 2021 14:46:03 -0400 Subject: [PATCH 36/79] Loosens the constraints around what can be in a new-repository-version. There are RPM repositories "in the wild" that violate Pulp's assumptions about what is 'legal' in an incoming repository version. This commit changes some fatal errors into log-warnings, along with some heuristics around how to get a 'reasonable' repository in the face of suboptimal data. To control activation of this behavior, RemoteArtifactSaver recognizes a ctor-keyword "fix_mismatched_remote_artifacts". This defaults to False; if provided as True, invokes the new behavior. This change also adds handling for an exception during repo-version-failure-cleanup that could result in losing error info. backports #8133 fixes #9130 (cherry picked from commit 349f2137d970f87190749b417cfa137dc0f99ff5) --- CHANGES/9130.bugfix | 2 + CHANGES/plugin_api/9130.bugfix | 5 +++ pulpcore/app/files.py | 33 ++++++++++----- pulpcore/app/models/publication.py | 9 +++- pulpcore/plugin/repo_version_utils.py | 3 +- pulpcore/plugin/stages/artifact_stages.py | 51 ++++++++++++++++++++++- pulpcore/tests/unit/test_files.py | 4 +- 7 files changed, 91 insertions(+), 16 deletions(-) create mode 100644 CHANGES/9130.bugfix create mode 100644 CHANGES/plugin_api/9130.bugfix diff --git a/CHANGES/9130.bugfix b/CHANGES/9130.bugfix new file mode 100644 index 0000000000..fc7f8354c7 --- /dev/null +++ b/CHANGES/9130.bugfix @@ -0,0 +1,2 @@ +In stages-pipeline and new-version sanity-checks, added full error-info on path-problems. +(backported from #8133) diff --git a/CHANGES/plugin_api/9130.bugfix b/CHANGES/plugin_api/9130.bugfix new file mode 100644 index 0000000000..762522cf01 --- /dev/null +++ b/CHANGES/plugin_api/9130.bugfix @@ -0,0 +1,5 @@ +Added kwarg to RemoteArtifactSaver init to allow enabling handling of rare error edge-case. + +`fix_mismatched_remote_artifacts=True` enables workaround for a failure-scenario that +(so far) is only encountered by pulp_rpm. Current behavior is the default. +(backported from #8133) diff --git a/pulpcore/app/files.py b/pulpcore/app/files.py index e01f846f02..c60bbc4977 100644 --- a/pulpcore/app/files.py +++ b/pulpcore/app/files.py @@ -128,21 +128,34 @@ def validate_file_paths(paths): overlap_error = _("The path for file '{path}' overlaps: {conflicts}") path_trie = StringTrie(separator="/") + dups = [] + overlaps = [] for path in paths: if path in path_trie: # path duplicates a path already in the trie - raise ValueError(_("Path is duplicated: {path}").format(path=path)) - - if path_trie.has_subtrie(path): + dups.append(path) + elif path_trie.has_subtrie(path): # overlap where path is 'a/b' and trie has 'a/b/c' conflicts = [item[0] for item in path_trie.items(prefix=path)] - raise ValueError(overlap_error.format(path=path, conflicts=(", ").join(conflicts))) - - prefixes = list(path_trie.prefixes(path)) - if prefixes: - # overlap where path is 'a/b/c' and trie has 'a/b' - conflicts = [prefix.key for prefix in prefixes] - raise ValueError(overlap_error.format(path=path, conflicts=(", ").join(conflicts))) + overlaps.append(overlap_error.format(path=path, conflicts=", ".join(conflicts))) + else: + prefixes = list(path_trie.prefixes(path)) + if prefixes: + # overlap where path is 'a/b/c' and trie has 'a/b' + conflicts = [prefix.key for prefix in prefixes] + overlaps.append(overlap_error.format(path=path, conflicts=", ".join(conflicts))) # if there are no overlaps, add it to our trie and continue path_trie[path] = True + + if dups or overlaps: + dups_msg = "" + overlaps_msg = "" + if dups: + dups_msg = _("Paths are duplicated: {paths}").format(paths=",".join(dups)) + if overlaps: + overlaps_msg = "\n".join(overlaps) + + raise ValueError( + _("Path errors found. {dups}\n{overlaps}").format(dups=dups_msg, overlaps=overlaps_msg) + ) diff --git a/pulpcore/app/models/publication.py b/pulpcore/app/models/publication.py index e3d8ac6f2e..2285861129 100644 --- a/pulpcore/app/models/publication.py +++ b/pulpcore/app/models/publication.py @@ -174,7 +174,14 @@ def __exit__(self, exc_type, exc_val, exc_tb): exc_tb (types.TracebackType): (optional) stack trace. """ if exc_val: - self.delete() + # If an exception got us here, the Publication we were trying to create is + # Bad, and we should delete the attempt. HOWEVER - some exceptions happen before we + # even get that far. In those cases, calling delete() results in a new not-very-useful + # exception being raised and reported to the user, rather than the actual problem. + try: + self.delete() + except Exception: + raise exc_val.with_traceback(exc_tb) else: try: self.finalize_new_publication() diff --git a/pulpcore/plugin/repo_version_utils.py b/pulpcore/plugin/repo_version_utils.py index 4d8b4a8f4f..ec77e8b628 100644 --- a/pulpcore/plugin/repo_version_utils.py +++ b/pulpcore/plugin/repo_version_utils.py @@ -119,11 +119,10 @@ def validate_version_paths(version): paths = ContentArtifact.objects.filter(content__pk__in=version.content).values_list( "relative_path", flat=True ) - try: validate_file_paths(paths) except ValueError as e: - raise ValueError(_("Cannot create repository version. {err}.").format(err=e)) + raise ValueError(_("Repository version errors : {err}").format(err=e)) def validate_repo_version(version): diff --git a/pulpcore/plugin/stages/artifact_stages.py b/pulpcore/plugin/stages/artifact_stages.py index c79f5a1d78..c7db7f36bd 100644 --- a/pulpcore/plugin/stages/artifact_stages.py +++ b/pulpcore/plugin/stages/artifact_stages.py @@ -262,6 +262,10 @@ class RemoteArtifactSaver(Stage): :class:`~pulpcore.plugin.stages.DeclarativeArtifact`. """ + def __init__(self, fix_mismatched_remote_artifacts=False, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fix_mismatched_remote_artifacts = fix_mismatched_remote_artifacts + async def run(self): """ The coroutine for this stage. @@ -326,8 +330,51 @@ def _needed_remote_artifacts(self, batch): if d_artifact.relative_path == content_artifact.relative_path: break else: - msg = _('No declared artifact with relative path "{rp}" for content "{c}"') - raise ValueError(msg.format(rp=d_artifact.relative_path, c=d_content.content)) + if self.fix_mismatched_remote_artifacts: + # We couldn't match an DeclarativeArtifact to a ContentArtifact by rel_path. + # If there are any paths available (i.e., other ContentArtifacts for this + # Artifact), complain to the logs, pick the rel_path from the last + # ContentArtifact we examined, and continue. + # + # If we can't find anything to choose from (can that even happen?), fail + # the process. + avail_paths = ",".join( + [ + ca.relative_path + for ca in d_content.content._remote_artifact_saver_cas + ] + ) + if avail_paths: + msg = _( + "No declared artifact with relative path '{rp}' for content '{c}'" + " from remote '{rname}'. Using last from available-paths : '{ap}'" + ) + log.warning( + msg.format( + rp=d_artifact.relative_path, + c=d_content.content.filename, + rname=d_artifact.remote.name, + ap=avail_paths, + ) + ) + d_artifact.relative_path = content_artifact.relative_path + else: + msg = _( + "No declared artifact with relative path '{rp}' for content '{c}'" + " from remote '{rname}', and no paths available." + ) + raise ValueError( + msg.format( + rp=d_artifact.relative_path, + c=d_content.content.filename, + rname=d_artifact.remote.name, + ) + ) + else: + msg = _('No declared artifact with relative path "{rp}" for content "{c}"') + raise ValueError( + msg.format(rp=d_artifact.relative_path, c=d_content.content) + ) for remote_artifact in content_artifact._remote_artifact_saver_ras: if remote_artifact.remote_id == d_artifact.remote.pk: diff --git a/pulpcore/tests/unit/test_files.py b/pulpcore/tests/unit/test_files.py index 0167776ca1..bc32059838 100644 --- a/pulpcore/tests/unit/test_files.py +++ b/pulpcore/tests/unit/test_files.py @@ -19,7 +19,9 @@ def test_dupes(self): Test for two duplicate paths. """ paths = ["a/b", "PULP_MANIFEST", "PULP_MANIFEST"] - with self.assertRaisesRegex(ValueError, "Path is duplicated: PULP_MANIFEST"): + with self.assertRaisesRegex( + ValueError, "Path errors found. Paths are duplicated: PULP_MANIFEST" + ): validate_file_paths(paths) def test_overlaps(self): From 79900fe8ead2cf5b879d2908d18b2ae6b2bcd15c Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Mon, 9 Aug 2021 14:48:30 -0400 Subject: [PATCH 37/79] Move files to artifact storage from WORKING_DIRECTORY. Copy files from all other sources. Since users are able to change the WORKING_DIRECTORY settings, this patch also ensures that we check that the WORKING_DIRECTORY and MEDIA_ROOT are on the same device. Django's FILE_UPLOAD_TEMP_DIR is now set to the value of WORKING_DIRECTORY. backports: #9146 https://pulp.plan.io/issues/9146/ fixes #9202 (cherry picked from commit c828ca5947f55e655e4020c55d92fc7b7ee01083) --- CHANGES/9202.bugfix | 3 +++ pulpcore/app/checks.py | 10 +++++----- pulpcore/app/models/storage.py | 9 +++++---- pulpcore/app/settings.py | 5 +++-- 4 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 CHANGES/9202.bugfix diff --git a/CHANGES/9202.bugfix b/CHANGES/9202.bugfix new file mode 100644 index 0000000000..4412f2ff4c --- /dev/null +++ b/CHANGES/9202.bugfix @@ -0,0 +1,3 @@ +Move files to artifact storage only when they originate from WORKING_DIRECTORY. +Copy files from all other sources. +(backported from #9146) diff --git a/pulpcore/app/checks.py b/pulpcore/app/checks.py index 4c9755caa9..975022c704 100644 --- a/pulpcore/app/checks.py +++ b/pulpcore/app/checks.py @@ -21,20 +21,20 @@ def storage_paths(app_configs, **kwargs): ) try: - upload_temp_dir_dev = Path(settings.FILE_UPLOAD_TEMP_DIR).stat().st_dev + working_dir_dev = Path(settings.WORKING_DIRECTORY).stat().st_dev except OSError: - upload_temp_dir_dev = None + working_dir_dev = None warnings.append( CheckWarning( - "Your FILE_UPLOAD_TEMP_DIR setting points to a path that does not exist.", + "Your WORKING_DIRECTORY setting points to a path that does not exist.", id="pulpcore.W002", ) ) - if media_root_dev and media_root_dev != upload_temp_dir_dev: + if media_root_dev and media_root_dev != working_dir_dev: warnings.append( CheckWarning( - "MEDIA_ROOT and FILE_UPLOAD_TEMP_DIR are on different filesystems. " + "MEDIA_ROOT and WORKING_DIRECTORY are on different filesystems. " "It is highly recommended that these live on the same filesystem", id="pulpcore.W003", ) diff --git a/pulpcore/app/models/storage.py b/pulpcore/app/models/storage.py index d558e82125..cd6fcb9d20 100644 --- a/pulpcore/app/models/storage.py +++ b/pulpcore/app/models/storage.py @@ -11,9 +11,10 @@ class FileSystem(FileSystemStorage): """ Django's FileSystemStorage with modified _save() and get_available_name behaviors - The _save() will check if the file is saved in DEPLOY_ROOT first. If it is, a move is used. This - will move all files created by the Downloaders and uploaded files from the user. If it is saved - in-memory or outside of DEPLOY_ROOT, the data is written/copied in chunks to the new location. + The _save() will check if the file is saved in WORKING_DIRECTORY first. If it is, a move is + used. This will move all files created by the Downloaders and uploaded files from the user. If + it is saved in-memory or outside of WORKING_DIRECTORY, the data is written/copied in chunks to + the new location. """ def get_available_name(self, name, max_length=None): @@ -63,7 +64,7 @@ def _save(self, name, content, max_length=None): try: if hasattr(content, "temporary_file_path") and content.temporary_file_path().startswith( - str(settings.DEPLOY_ROOT) + str(settings.WORKING_DIRECTORY) ): file_move_safe(content.temporary_file_path(), full_path) else: diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py index 578007f788..8a544803a8 100644 --- a/pulpcore/app/settings.py +++ b/pulpcore/app/settings.py @@ -46,8 +46,9 @@ DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem" -FILE_UPLOAD_TEMP_DIR = DEPLOY_ROOT / "tmp" -WORKING_DIRECTORY = FILE_UPLOAD_TEMP_DIR +WORKING_DIRECTORY = DEPLOY_ROOT / "tmp" +FILE_UPLOAD_TEMP_DIR = WORKING_DIRECTORY + CHUNKED_UPLOAD_DIR = "upload" # List of upload handler classes to be applied in order. From a16955a760b6035efd7189b234e2bdb3429558d9 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Mon, 9 Aug 2021 14:49:43 -0400 Subject: [PATCH 38/79] Handle a potential failure during cache invalidation A completed publication might not exist when we try to delete it. backports: #9174 https://pulp.plan.io/issues/9174 fixes #9179 (cherry picked from commit 248b4d467aa9aad8c4e6787ef577c99297940bf9) --- CHANGES/9179.misc | 2 ++ pulpcore/app/models/publication.py | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 CHANGES/9179.misc diff --git a/CHANGES/9179.misc b/CHANGES/9179.misc new file mode 100644 index 0000000000..954caa51e7 --- /dev/null +++ b/CHANGES/9179.misc @@ -0,0 +1,2 @@ +Fix an error that could sometimes occur during deletion of an incomplete publication. +(backported from #9174) diff --git a/pulpcore/app/models/publication.py b/pulpcore/app/models/publication.py index 2285861129..d1990ee316 100644 --- a/pulpcore/app/models/publication.py +++ b/pulpcore/app/models/publication.py @@ -130,11 +130,21 @@ def delete(self, **kwargs): # Find any publications being served directly base_paths = self.distribution_set.values_list("base_path", flat=True) # Find any publications being served indirectly by auto-distribute feature - versions = self.repository.versions.all() - pubs = Publication.objects.filter(repository_version__in=versions, complete=True) - publication = pubs.latest("repository_version", "pulp_created") - if self.pk == publication.pk: - base_paths |= self.repository.distributions.values_list("base_path", flat=True) + # It's possible for errors to occur before any publication has been completed, + # so we need to handle the case when no Publication exists. + try: + versions = self.repository.versions.all() + pubs = Publication.objects.filter( + repository_version__in=versions, complete=True + ) + publication = pubs.latest("repository_version", "pulp_created") + if self.pk == publication.pk: + base_paths |= self.repository.distributions.values_list( + "base_path", flat=True + ) + except Publication.DoesNotExist: + pass + # Invalidate cache for all distributions serving this publication if base_paths: Cache().delete(base_key=base_paths) From 3cc35436b1225a25f14ad3b195e26c705d22784e Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 10 Aug 2021 16:44:22 +0000 Subject: [PATCH 39/79] Building changelog for 3.14.4 [noissue] --- CHANGES.rst | 50 +++++++++++++++++++++++++++++++++ CHANGES/9126.bugfix | 3 -- CHANGES/9130.bugfix | 2 -- CHANGES/9179.misc | 2 -- CHANGES/9202.bugfix | 3 -- CHANGES/plugin_api/9126.feature | 3 -- CHANGES/plugin_api/9130.bugfix | 5 ---- 7 files changed, 50 insertions(+), 18 deletions(-) delete mode 100644 CHANGES/9126.bugfix delete mode 100644 CHANGES/9130.bugfix delete mode 100644 CHANGES/9179.misc delete mode 100644 CHANGES/9202.bugfix delete mode 100644 CHANGES/plugin_api/9126.feature delete mode 100644 CHANGES/plugin_api/9130.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index 6bd02cb3fb..a3b7231c55 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,56 @@ Changelog .. towncrier release notes start +3.14.4 (2021-08-10) +=================== +REST API +-------- + +Bugfixes +~~~~~~~~ + +- Unpublished content can no longer be accessed from content app if publication based-plugin has + defined their distributions as publication serving + (backported from #8870) + `#9126 `_ +- In stages-pipeline and new-version sanity-checks, added full error-info on path-problems. + (backported from #8133) + `#9130 `_ +- Move files to artifact storage only when they originate from WORKING_DIRECTORY. + Copy files from all other sources. + (backported from #9146) + `#9202 `_ + + +Misc +~~~~ + +- `#9179 `_ + + +Plugin API +---------- + +Features +~~~~~~~~ + +- Distribution model has a new boolean class variable ``SERVE_FROM_PUBLICATION`` for plugins to declare + whether their distributions serve from publications or directly from repository versions + (backported from #8870) + `#9126 `_ + + +Bugfixes +~~~~~~~~ + +- Added kwarg to RemoteArtifactSaver init to allow enabling handling of rare error edge-case. + + `fix_mismatched_remote_artifacts=True` enables workaround for a failure-scenario that + (so far) is only encountered by pulp_rpm. Current behavior is the default. + (backported from #8133) + `#9130 `_ + + 3.14.3 (2021-07-23) =================== REST API diff --git a/CHANGES/9126.bugfix b/CHANGES/9126.bugfix deleted file mode 100644 index a16d959059..0000000000 --- a/CHANGES/9126.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Unpublished content can no longer be accessed from content app if publication based-plugin has -defined their distributions as publication serving -(backported from #8870) diff --git a/CHANGES/9130.bugfix b/CHANGES/9130.bugfix deleted file mode 100644 index fc7f8354c7..0000000000 --- a/CHANGES/9130.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -In stages-pipeline and new-version sanity-checks, added full error-info on path-problems. -(backported from #8133) diff --git a/CHANGES/9179.misc b/CHANGES/9179.misc deleted file mode 100644 index 954caa51e7..0000000000 --- a/CHANGES/9179.misc +++ /dev/null @@ -1,2 +0,0 @@ -Fix an error that could sometimes occur during deletion of an incomplete publication. -(backported from #9174) diff --git a/CHANGES/9202.bugfix b/CHANGES/9202.bugfix deleted file mode 100644 index 4412f2ff4c..0000000000 --- a/CHANGES/9202.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Move files to artifact storage only when they originate from WORKING_DIRECTORY. -Copy files from all other sources. -(backported from #9146) diff --git a/CHANGES/plugin_api/9126.feature b/CHANGES/plugin_api/9126.feature deleted file mode 100644 index b556177acf..0000000000 --- a/CHANGES/plugin_api/9126.feature +++ /dev/null @@ -1,3 +0,0 @@ -Distribution model has a new boolean class variable ``SERVE_FROM_PUBLICATION`` for plugins to declare -whether their distributions serve from publications or directly from repository versions -(backported from #8870) diff --git a/CHANGES/plugin_api/9130.bugfix b/CHANGES/plugin_api/9130.bugfix deleted file mode 100644 index 762522cf01..0000000000 --- a/CHANGES/plugin_api/9130.bugfix +++ /dev/null @@ -1,5 +0,0 @@ -Added kwarg to RemoteArtifactSaver init to allow enabling handling of rare error edge-case. - -`fix_mismatched_remote_artifacts=True` enables workaround for a failure-scenario that -(so far) is only encountered by pulp_rpm. Current behavior is the default. -(backported from #8133) From c24025df87ebe24ead08edc9fde07a8ed5bc4976 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 10 Aug 2021 16:44:25 +0000 Subject: [PATCH 40/79] Release 3.14.4 Redmine Query: https://pulp.plan.io/issues?set_filter=1&status_id=*&issue_id=9202,9126,9130,9179 Redmine Milestone: https://pulp.plan.io/versions/268.json [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f7d952c435..7c4c490de0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.4.dev +current_version = 3.14.4 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index cc1ca31d26..8a9534bef4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.4.dev" +version = "3.14.4" # The full version, including alpha/beta/rc tags. -release = "3.14.4.dev" +release = "3.14.4" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 927fc4a11a..bb717bb239 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.4.dev" + version = "3.14.4" def ready(self): super().ready() diff --git a/setup.py b/setup.py index b36420a758..ab4d557bfb 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.4.dev", + version="3.14.4", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 091de329a3667e45eee4ec3c25fdddf594ab4c41 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 10 Aug 2021 16:44:25 +0000 Subject: [PATCH 41/79] Bump to 3.14.5.dev [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7c4c490de0..1d18b429bc 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.4 +current_version = 3.14.5.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 8a9534bef4..2b5091a792 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.4" +version = "3.14.5.dev" # The full version, including alpha/beta/rc tags. -release = "3.14.4" +release = "3.14.5.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index bb717bb239..0903fa53ef 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.4" + version = "3.14.5.dev" def ready(self): super().ready() diff --git a/setup.py b/setup.py index ab4d557bfb..dcac6fc1d5 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.4", + version="3.14.5.dev", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 1205007283b3185d5a5bb4a4c2b29aa7b36a945b Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Fri, 20 Aug 2021 20:24:57 -0400 Subject: [PATCH 42/79] Removing OpenAPI blank enums backports #9056 fixes #9265 (cherry picked from commit eab862a60094f2c38622435196c28ff800700d15) --- CHANGES/9265.misc | 2 ++ pulpcore/app/settings.py | 1 + 2 files changed, 3 insertions(+) create mode 100644 CHANGES/9265.misc diff --git a/CHANGES/9265.misc b/CHANGES/9265.misc new file mode 100644 index 0000000000..9aecf40585 --- /dev/null +++ b/CHANGES/9265.misc @@ -0,0 +1,2 @@ +Removing OpenAPI blank enums +(backported from #9056) diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py index 8a544803a8..89a15e0b95 100644 --- a/pulpcore/app/settings.py +++ b/pulpcore/app/settings.py @@ -250,6 +250,7 @@ "SERVE_URLCONF": ROOT_URLCONF, "DEFAULT_GENERATOR_CLASS": "pulpcore.openapi.PulpSchemaGenerator", "DEFAULT_SCHEMA_CLASS": "pulpcore.openapi.PulpAutoSchema", + "ENUM_ADD_EXPLICIT_BLANK_NULL_CHOICE": False, "COMPONENT_SPLIT_REQUEST": True, "COMPONENT_NO_READ_ONLY_REQUIRED": True, "GENERIC_ADDITIONAL_PROPERTIES": None, From 699e17ac6e3e6dd7130614f7646ab1e360508fda Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Fri, 20 Aug 2021 20:33:05 -0400 Subject: [PATCH 43/79] Compatibility patch for backport #9268 Adding a small portion of a refactor from 81f6a78175bb to facilitate the backport of another patch. [noissue] --- pulpcore/app/models/content.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pulpcore/app/models/content.py b/pulpcore/app/models/content.py index a0fdd317ad..e436e5b39f 100644 --- a/pulpcore/app/models/content.py +++ b/pulpcore/app/models/content.py @@ -139,6 +139,10 @@ def delete(self, *args, **kwargs): self.file.delete(save=False) +class ArtifactManager(BulkCreateManager): + pass + + class Artifact(HandleTempFilesMixin, BaseModel): """ A file associated with a piece of content. @@ -183,7 +187,7 @@ def storage_path(self, name): sha512 = models.CharField(max_length=128, null=True, unique=True, db_index=True) timestamp_of_interest = models.DateTimeField(auto_now=True) - objects = BulkCreateManager() + objects = ArtifactManager() # All available digest fields ordered by algorithm strength. DIGEST_FIELDS = _DIGEST_FIELDS @@ -426,6 +430,10 @@ def init_and_validate(file, expected_digests=None, expected_size=None): return PulpTemporaryFile(file=file) +class ContentManager(BulkCreateManager): + pass + + class Content(MasterModel, QueryMixin): """ A piece of managed content. @@ -446,7 +454,7 @@ class Content(MasterModel, QueryMixin): _artifacts = models.ManyToManyField(Artifact, through="ContentArtifact") timestamp_of_interest = models.DateTimeField(auto_now=True) - objects = BulkCreateManager() + objects = ContentManager() class Meta: verbose_name_plural = "content" From 966122bf17ed0d48b3a33c6a58934e0ef2a10524 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Fri, 20 Aug 2021 20:39:57 -0400 Subject: [PATCH 44/79] Add touch to QuerySets of artifacts or content backports #9234 fixes #9268 (cherry picked from commit aafe3766dd24426a58cd02d73d8834bc84f7ab50) --- CHANGES/plugin_api/9268.feature | 2 ++ pulpcore/app/models/content.py | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 CHANGES/plugin_api/9268.feature diff --git a/CHANGES/plugin_api/9268.feature b/CHANGES/plugin_api/9268.feature new file mode 100644 index 0000000000..6658574bda --- /dev/null +++ b/CHANGES/plugin_api/9268.feature @@ -0,0 +1,2 @@ +Added ``touch`` to Artifact and Content query sets for bulk operation. +(backported from #9234) diff --git a/pulpcore/app/models/content.py b/pulpcore/app/models/content.py index e436e5b39f..936abe7cc1 100644 --- a/pulpcore/app/models/content.py +++ b/pulpcore/app/models/content.py @@ -17,6 +17,7 @@ from django.core.files.storage import default_storage from django.db import IntegrityError, models, transaction from django.forms.models import model_to_dict +from django.utils.timezone import now from django_lifecycle import BEFORE_UPDATE, BEFORE_SAVE, hook from pulpcore.constants import ALL_KNOWN_CONTENT_CHECKSUMS @@ -90,6 +91,18 @@ def bulk_get_or_create(self, objs, batch_size=None): return objs +class BulkTouchQuerySet(models.QuerySet): + """ + A query set that provides ``touch()``. + """ + + def touch(self): + """ + Update the ``timestamp_of_interest`` on all objects of the query. + """ + return self.update(timestamp_of_interest=now()) + + class QueryMixin: """ A mixin that provides models with querying utilities. @@ -187,7 +200,7 @@ def storage_path(self, name): sha512 = models.CharField(max_length=128, null=True, unique=True, db_index=True) timestamp_of_interest = models.DateTimeField(auto_now=True) - objects = ArtifactManager() + objects = ArtifactManager.from_queryset(BulkTouchQuerySet)() # All available digest fields ordered by algorithm strength. DIGEST_FIELDS = _DIGEST_FIELDS @@ -454,7 +467,7 @@ class Content(MasterModel, QueryMixin): _artifacts = models.ManyToManyField(Artifact, through="ContentArtifact") timestamp_of_interest = models.DateTimeField(auto_now=True) - objects = ContentManager() + objects = ContentManager.from_queryset(BulkTouchQuerySet)() class Meta: verbose_name_plural = "content" From f5f6713ac92f24cf71df13d80d7a686e2d745cea Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Fri, 20 Aug 2021 20:25:07 -0400 Subject: [PATCH 45/79] Use bulk touch() when processing Artifacts and Content This patch only addresses the inefficiencies of the Stages API. Another patch is needed to address the inefficiency of the repository version modify operation. backports: #9243 fixes #9264 (cherry picked from commit 7a41c18b9c2c30ae31c8d248162f42cb6c63c529) --- CHANGES/9264.bugfix | 2 ++ pulpcore/plugin/stages/artifact_stages.py | 11 ++--------- pulpcore/plugin/stages/content_stages.py | 11 +++-------- 3 files changed, 7 insertions(+), 17 deletions(-) create mode 100644 CHANGES/9264.bugfix diff --git a/CHANGES/9264.bugfix b/CHANGES/9264.bugfix new file mode 100644 index 0000000000..4e4f7d822d --- /dev/null +++ b/CHANGES/9264.bugfix @@ -0,0 +1,2 @@ +Fixed repository sync performance regression introduced in pulpcore 3.14. +(backported from #9243) diff --git a/pulpcore/plugin/stages/artifact_stages.py b/pulpcore/plugin/stages/artifact_stages.py index c7db7f36bd..9eda6d1d69 100644 --- a/pulpcore/plugin/stages/artifact_stages.py +++ b/pulpcore/plugin/stages/artifact_stages.py @@ -3,7 +3,6 @@ from gettext import gettext as _ import logging -from django.db import DatabaseError from django.db.models import Prefetch, prefetch_related_objects from pulpcore.plugin.exceptions import UnsupportedDigestValidationError @@ -87,6 +86,7 @@ async def run(self): for digest_type, digests in artifact_digests_by_type.items(): query_params = {"{attr}__in".format(attr=digest_type): digests} existing_artifacts = Artifact.objects.filter(**query_params).only(digest_type) + existing_artifacts.touch() for d_content in batch: for d_artifact in d_content.d_artifacts: artifact_digest = getattr(d_artifact.artifact, digest_type) @@ -94,15 +94,8 @@ async def run(self): for result in existing_artifacts: result_digest = getattr(result, digest_type) if result_digest == artifact_digest: - try: - result.touch() - except DatabaseError: - # update failed so leave artifact empty to create it later - pass - else: - d_artifact.artifact = result + d_artifact.artifact = result break - for d_content in batch: await self.put(d_content) diff --git a/pulpcore/plugin/stages/content_stages.py b/pulpcore/plugin/stages/content_stages.py index cf79f717ef..66a9f2671b 100644 --- a/pulpcore/plugin/stages/content_stages.py +++ b/pulpcore/plugin/stages/content_stages.py @@ -1,7 +1,7 @@ from collections import defaultdict from django.core.exceptions import ObjectDoesNotExist -from django.db import DatabaseError, IntegrityError, transaction +from django.db import IntegrityError, transaction from django.db.models import Q from pulpcore.plugin.models import Content, ContentArtifact, ProgressReport @@ -48,15 +48,10 @@ async def run(self): d_content_by_nat_key[d_content.content.natural_key()].append(d_content) for model_type in content_q_by_type.keys(): + model_type.objects.filter(content_q_by_type[model_type]).touch() for result in model_type.objects.filter(content_q_by_type[model_type]).iterator(): for d_content in d_content_by_nat_key[result.natural_key()]: - try: - result.touch() - except DatabaseError: - # update failed so leave content empty to create it later - pass - else: - d_content.content = result + d_content.content = result for d_content in batch: await self.put(d_content) From 6c44ecc1919c0bc01ef4617cfed376bfefb8dada Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Fri, 20 Aug 2021 20:43:35 -0400 Subject: [PATCH 46/79] Content app now properly sets Content-Type header for S3 backports: #9216 fixes #9244 (cherry picked from commit feede7bb08b1e3107766ce534433cb7867fe52bd) --- CHANGES/9244.bugfix | 2 ++ pulpcore/content/handler.py | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 CHANGES/9244.bugfix diff --git a/CHANGES/9244.bugfix b/CHANGES/9244.bugfix new file mode 100644 index 0000000000..b1f7ca1c42 --- /dev/null +++ b/CHANGES/9244.bugfix @@ -0,0 +1,2 @@ +Content app now properly sets Content-Type header for artifacts being served from S3 +(backported from #9216) diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py index 03631328b2..c740cd1628 100644 --- a/pulpcore/content/handler.py +++ b/pulpcore/content/handler.py @@ -788,6 +788,8 @@ async def _serve_content_artifact(self, content_artifact, headers): elif settings.DEFAULT_FILE_STORAGE == "storages.backends.s3boto3.S3Boto3Storage": content_disposition = f"attachment;filename={content_artifact.relative_path}" parameters = {"ResponseContentDisposition": content_disposition} + if headers.get("Content-Type"): + parameters["ResponseContentType"] = headers.get("Content-Type") url = URL( artifact_file.storage.url(artifact_file.name, parameters=parameters), encoded=True ) From 11e95dea649f3ce6042e7e1038b9fc026b92c273 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Aug 2021 20:12:21 +0000 Subject: [PATCH 47/79] Dependency version bump [noissue] --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7bb30f4a62..f3d5784ddd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,8 +16,8 @@ djangorestframework~=3.12.4 djangorestframework-queryfields~=1.0.0 drf-access-policy~=0.9.0 drf-nested-routers==0.93.3 -drf-spectacular==0.17.2 -dynaconf~=3.1.4 +drf-spectacular==0.17.3 +dynaconf~=3.1.5 gunicorn~=20.1.0 jinja2 pygtrie~=2.4.2 From 6fee19c08c0e10b5820e4ed259130e1554bd1969 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Mon, 23 Aug 2021 11:38:34 -0400 Subject: [PATCH 48/79] Fix issue where plugins with custom content managers can't use touch() backports: #9269 https://pulp.plan.io/issues/9269 fixes #9273 (cherry picked from commit 2af65cf90e66fab222293fd8f809d84877c6bf6b) --- CHANGES/plugin_api/9273.feature | 2 ++ pulpcore/app/models/__init__.py | 1 + pulpcore/app/models/content.py | 5 ++++- pulpcore/plugin/models/__init__.py | 1 + pulpcore/plugin/stages/content_stages.py | 19 ++++++++++++++++--- 5 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 CHANGES/plugin_api/9273.feature diff --git a/CHANGES/plugin_api/9273.feature b/CHANGES/plugin_api/9273.feature new file mode 100644 index 0000000000..dd841f75dc --- /dev/null +++ b/CHANGES/plugin_api/9273.feature @@ -0,0 +1,2 @@ +Added `ContentManager` to the plugin API - all subclasses of `Content` that add their own custom manager should have the manager subclass `ContentManager`. +(backported from #9269) diff --git a/pulpcore/app/models/__init__.py b/pulpcore/app/models/__init__.py index ed00d2e029..6c184be32c 100644 --- a/pulpcore/app/models/__init__.py +++ b/pulpcore/app/models/__init__.py @@ -12,6 +12,7 @@ Artifact, AsciiArmoredDetachedSigningService, Content, + ContentManager, ContentArtifact, PulpTemporaryFile, RemoteArtifact, diff --git a/pulpcore/app/models/content.py b/pulpcore/app/models/content.py index 936abe7cc1..64aae6ab0b 100644 --- a/pulpcore/app/models/content.py +++ b/pulpcore/app/models/content.py @@ -447,6 +447,9 @@ class ContentManager(BulkCreateManager): pass +ContentManager = ContentManager.from_queryset(BulkTouchQuerySet) + + class Content(MasterModel, QueryMixin): """ A piece of managed content. @@ -467,7 +470,7 @@ class Content(MasterModel, QueryMixin): _artifacts = models.ManyToManyField(Artifact, through="ContentArtifact") timestamp_of_interest = models.DateTimeField(auto_now=True) - objects = ContentManager.from_queryset(BulkTouchQuerySet)() + objects = ContentManager() class Meta: verbose_name_plural = "content" diff --git a/pulpcore/plugin/models/__init__.py b/pulpcore/plugin/models/__init__.py index e5d1e30a25..c268df3018 100644 --- a/pulpcore/plugin/models/__init__.py +++ b/pulpcore/plugin/models/__init__.py @@ -10,6 +10,7 @@ BaseModel, Content, ContentArtifact, + ContentManager, ContentGuard, CreatedResource, Distribution, diff --git a/pulpcore/plugin/stages/content_stages.py b/pulpcore/plugin/stages/content_stages.py index 66a9f2671b..2bb75b109d 100644 --- a/pulpcore/plugin/stages/content_stages.py +++ b/pulpcore/plugin/stages/content_stages.py @@ -47,9 +47,22 @@ async def run(self): content_q_by_type[model_type] = content_q_by_type[model_type] | unit_q d_content_by_nat_key[d_content.content.natural_key()].append(d_content) - for model_type in content_q_by_type.keys(): - model_type.objects.filter(content_q_by_type[model_type]).touch() - for result in model_type.objects.filter(content_q_by_type[model_type]).iterator(): + for model_type, content_q in content_q_by_type.items(): + try: + model_type.objects.filter(content_q).touch() + except AttributeError: + from pulpcore.app.loggers import deprecation_logger + from gettext import gettext as _ + + deprecation_logger.warning( + _( + "As of pulpcore 3.14.5, plugins which declare custom ORM managers on " + "their content classes should have those managers inherit from " + "pulpcore.plugin.models.ContentManager. This will become a hard error " + "in the future." + ) + ) + for result in model_type.objects.filter(content_q).iterator(): for d_content in d_content_by_nat_key[result.natural_key()]: d_content.content = result From fbec2956985312f7c31122c68999de5b2508f392 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Mon, 23 Aug 2021 14:16:21 -0400 Subject: [PATCH 49/79] Dropping some items from the changelog [noissue] --- CHANGES/plugin_api/{9268.feature => 9268.misc} | 0 CHANGES/plugin_api/{9273.feature => 9273.misc} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename CHANGES/plugin_api/{9268.feature => 9268.misc} (100%) rename CHANGES/plugin_api/{9273.feature => 9273.misc} (100%) diff --git a/CHANGES/plugin_api/9268.feature b/CHANGES/plugin_api/9268.misc similarity index 100% rename from CHANGES/plugin_api/9268.feature rename to CHANGES/plugin_api/9268.misc diff --git a/CHANGES/plugin_api/9273.feature b/CHANGES/plugin_api/9273.misc similarity index 100% rename from CHANGES/plugin_api/9273.feature rename to CHANGES/plugin_api/9273.misc From 082f4ee1bbd7902d0fbeb0ff3d0f824fe7b46cd6 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Tue, 24 Aug 2021 08:45:53 -0400 Subject: [PATCH 50/79] Reset a db connection in content app distribution matching code It's a workaround for yet another `connection already closed` issue. backports #9275 https://pulp.plan.io/issues/9275 https://pulp.plan.io/issues/9276 is filed to investigate that issue more. fixes #9282 (cherry picked from commit d603476240edb942e1505c2deea5dc697fedd4a2) --- CHANGES/9282.bugfix | 2 ++ pulpcore/content/handler.py | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 CHANGES/9282.bugfix diff --git a/CHANGES/9282.bugfix b/CHANGES/9282.bugfix new file mode 100644 index 0000000000..89c13f803a --- /dev/null +++ b/CHANGES/9282.bugfix @@ -0,0 +1,2 @@ +Fixed another occurence of the HTTP 500 error and `connection already closed` in the logs while accessing content. +(backported from #9275) diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py index c740cd1628..7c9a1e3dad 100644 --- a/pulpcore/content/handler.py +++ b/pulpcore/content/handler.py @@ -246,6 +246,8 @@ def _match_distribution(cls, path): Raises: PathNotResolved: when not matched. """ + cls._reset_db_connection() + base_paths = cls._base_paths(path) if cls.distribution_model is None: try: From f36ffd043b5458e1b8d787e9f8d05e157a2cc6f5 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Tue, 24 Aug 2021 10:42:57 -0400 Subject: [PATCH 51/79] Properly update present ContentAtifacts after immediate sync backports: #9101 fixes #9261 (cherry picked from commit c2b732e55174e022d4f3e2a51d601b0ae71732aa) --- CHANGES/9261.bugfix | 2 + pulpcore/plugin/stages/content_stages.py | 43 ++++++++--- .../functional/api/using_plugin/test_sync.py | 73 +++++++++++++++++++ 3 files changed, 107 insertions(+), 11 deletions(-) create mode 100644 CHANGES/9261.bugfix create mode 100644 pulpcore/tests/functional/api/using_plugin/test_sync.py diff --git a/CHANGES/9261.bugfix b/CHANGES/9261.bugfix new file mode 100644 index 0000000000..6862e31d3c --- /dev/null +++ b/CHANGES/9261.bugfix @@ -0,0 +1,2 @@ +Artifacts are now being properly updated for Content after switching from 'on_demand' to 'immediate'. +(backported from #9101) diff --git a/pulpcore/plugin/stages/content_stages.py b/pulpcore/plugin/stages/content_stages.py index 2bb75b109d..f919d199d2 100644 --- a/pulpcore/plugin/stages/content_stages.py +++ b/pulpcore/plugin/stages/content_stages.py @@ -98,6 +98,9 @@ async def run(self): """ async for batch in self.batches(): content_artifact_bulk = [] + to_update_ca_query = ContentArtifact.objects.none() + to_update_ca_bulk = [] + to_update_ca_artifact = {} with transaction.atomic(): await self._pre_save(batch) @@ -115,19 +118,37 @@ async def run(self): ) except ObjectDoesNotExist: raise e + else: + for d_artifact in d_content.d_artifacts: + if not d_artifact.artifact._state.adding: + artifact = d_artifact.artifact + else: + # set to None for on-demand synced artifacts + artifact = None + content_artifact = ContentArtifact( + content=d_content.content, + artifact=artifact, + relative_path=d_artifact.relative_path, + ) + content_artifact_bulk.append(content_artifact) continue - for d_artifact in d_content.d_artifacts: - if not d_artifact.artifact._state.adding: - artifact = d_artifact.artifact - else: - # set to None for on-demand synced artifacts - artifact = None - content_artifact = ContentArtifact( - content=d_content.content, - artifact=artifact, - relative_path=d_artifact.relative_path, + # When the Content already exists, check if ContentArtifacts need to be updated + for d_artifact in d_content.d_artifacts: + if not d_artifact.artifact._state.adding: + # the artifact is already present in the database; update references + # Creating one large query and one large dictionary + to_update_ca_query |= ContentArtifact.objects.filter( + content=d_content.content, relative_path=d_artifact.relative_path ) - content_artifact_bulk.append(content_artifact) + key = (d_content.content.pk, d_artifact.relative_path) + to_update_ca_artifact[key] = d_artifact.artifact + # Query db once and update each object in memory for bulk_update call + for content_artifact in to_update_ca_query.iterator(): + key = (content_artifact.content_id, content_artifact.relative_path) + # Maybe remove dict elements after to reduce memory? + content_artifact.artifact = to_update_ca_artifact[key] + to_update_ca_bulk.append(content_artifact) + ContentArtifact.objects.bulk_update(to_update_ca_bulk, ["artifact"]) ContentArtifact.objects.bulk_get_or_create(content_artifact_bulk) await self._post_save(batch) for declarative_content in batch: diff --git a/pulpcore/tests/functional/api/using_plugin/test_sync.py b/pulpcore/tests/functional/api/using_plugin/test_sync.py new file mode 100644 index 0000000000..be460c9e55 --- /dev/null +++ b/pulpcore/tests/functional/api/using_plugin/test_sync.py @@ -0,0 +1,73 @@ +from pulp_smash.pulp3.bindings import delete_orphans, monitor_task, PulpTestCase +from pulp_smash.pulp3.utils import gen_repo + +from pulpcore.tests.functional.api.using_plugin.utils import ( + gen_file_client, + gen_file_remote, +) +from pulpcore.client.pulp_file import ( + ContentFilesApi, + RepositorySyncURL, + RepositoriesFileApi, + RemotesFileApi, +) +from pulpcore.tests.functional.api.using_plugin.utils import ( # noqa:F401 + set_up_module as setUpModule, +) + + +class MultiplePolicySyncTestCase(PulpTestCase): + """ + This test ensures that content artifacts are properly updated when syncing multiple + times with different policies, specifically from 'on_demand' to 'immediate' + + This test targets the following issue: + * `Pulp #9101 `_ + """ + + @classmethod + def setUpClass(cls): + """Clean out Pulp before testing.""" + delete_orphans() + client = gen_file_client() + cls.cont_api = ContentFilesApi(client) + cls.repo_api = RepositoriesFileApi(client) + cls.remote_api = RemotesFileApi(client) + + def tearDown(self): + """Clean up Pulp after testing.""" + self.doCleanups() + delete_orphans() + + def test_ondemand_to_immediate_sync(self): + """Checks that content artifacts are updated following on-demand -> immediate sync.""" + # Ensure that no content is present + content_response = self.cont_api.list(limit=1) + if content_response.count > 0: + self.skipTest("Please remove all file content before running this test") + + # Create and sync repo w/ on_demand policy + repo = self.repo_api.create(gen_repo()) + remote = self.remote_api.create(gen_file_remote(policy="on_demand")) + body = RepositorySyncURL(remote=remote.pulp_href) + monitor_task(self.repo_api.sync(repo.pulp_href, body).task) + self.addCleanup(self.repo_api.delete, repo.pulp_href) + self.addCleanup(self.remote_api.delete, remote.pulp_href) + + # Check content is present, but no artifacts are there + content_response = self.cont_api.list() + self.assertEqual(content_response.count, 3) + for content in content_response.results: + self.assertEqual(content.artifact, None) + + # Sync again w/ immediate policy + remote = self.remote_api.create(gen_file_remote()) + body = RepositorySyncURL(remote=remote.pulp_href) + monitor_task(self.repo_api.sync(repo.pulp_href, body).task) + self.addCleanup(self.remote_api.delete, remote.pulp_href) + + # Check content is still present, but artifacts are now there + content_response = self.cont_api.list() + self.assertEqual(content_response.count, 3) + for content in content_response.results: + self.assertNotEqual(content.artifact, None) From 63b37b11655f6490cd6fc7e22275f907f4f7fd24 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 24 Aug 2021 17:46:49 +0000 Subject: [PATCH 52/79] Building changelog for 3.14.5 [noissue] --- CHANGES.rst | 37 ++++++++++++++++++++++++++++++++++++ CHANGES/9244.bugfix | 2 -- CHANGES/9261.bugfix | 2 -- CHANGES/9264.bugfix | 2 -- CHANGES/9265.misc | 2 -- CHANGES/9282.bugfix | 2 -- CHANGES/plugin_api/9268.misc | 2 -- CHANGES/plugin_api/9273.misc | 2 -- 8 files changed, 37 insertions(+), 14 deletions(-) delete mode 100644 CHANGES/9244.bugfix delete mode 100644 CHANGES/9261.bugfix delete mode 100644 CHANGES/9264.bugfix delete mode 100644 CHANGES/9265.misc delete mode 100644 CHANGES/9282.bugfix delete mode 100644 CHANGES/plugin_api/9268.misc delete mode 100644 CHANGES/plugin_api/9273.misc diff --git a/CHANGES.rst b/CHANGES.rst index a3b7231c55..5bd7f304b3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,43 @@ Changelog .. towncrier release notes start +3.14.5 (2021-08-24) +=================== +REST API +-------- + +Bugfixes +~~~~~~~~ + +- Content app now properly sets Content-Type header for artifacts being served from S3 + (backported from #9216) + `#9244 `_ +- Artifacts are now being properly updated for Content after switching from 'on_demand' to 'immediate'. + (backported from #9101) + `#9261 `_ +- Fixed repository sync performance regression introduced in pulpcore 3.14. + (backported from #9243) + `#9264 `_ +- Fixed another occurence of the HTTP 500 error and `connection already closed` in the logs while accessing content. + (backported from #9275) + `#9282 `_ + + +Misc +~~~~ + +- `#9265 `_ + + +Plugin API +---------- + +Misc +~~~~ + +- `#9268 `_, `#9273 `_ + + 3.14.4 (2021-08-10) =================== REST API diff --git a/CHANGES/9244.bugfix b/CHANGES/9244.bugfix deleted file mode 100644 index b1f7ca1c42..0000000000 --- a/CHANGES/9244.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Content app now properly sets Content-Type header for artifacts being served from S3 -(backported from #9216) diff --git a/CHANGES/9261.bugfix b/CHANGES/9261.bugfix deleted file mode 100644 index 6862e31d3c..0000000000 --- a/CHANGES/9261.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Artifacts are now being properly updated for Content after switching from 'on_demand' to 'immediate'. -(backported from #9101) diff --git a/CHANGES/9264.bugfix b/CHANGES/9264.bugfix deleted file mode 100644 index 4e4f7d822d..0000000000 --- a/CHANGES/9264.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed repository sync performance regression introduced in pulpcore 3.14. -(backported from #9243) diff --git a/CHANGES/9265.misc b/CHANGES/9265.misc deleted file mode 100644 index 9aecf40585..0000000000 --- a/CHANGES/9265.misc +++ /dev/null @@ -1,2 +0,0 @@ -Removing OpenAPI blank enums -(backported from #9056) diff --git a/CHANGES/9282.bugfix b/CHANGES/9282.bugfix deleted file mode 100644 index 89c13f803a..0000000000 --- a/CHANGES/9282.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed another occurence of the HTTP 500 error and `connection already closed` in the logs while accessing content. -(backported from #9275) diff --git a/CHANGES/plugin_api/9268.misc b/CHANGES/plugin_api/9268.misc deleted file mode 100644 index 6658574bda..0000000000 --- a/CHANGES/plugin_api/9268.misc +++ /dev/null @@ -1,2 +0,0 @@ -Added ``touch`` to Artifact and Content query sets for bulk operation. -(backported from #9234) diff --git a/CHANGES/plugin_api/9273.misc b/CHANGES/plugin_api/9273.misc deleted file mode 100644 index dd841f75dc..0000000000 --- a/CHANGES/plugin_api/9273.misc +++ /dev/null @@ -1,2 +0,0 @@ -Added `ContentManager` to the plugin API - all subclasses of `Content` that add their own custom manager should have the manager subclass `ContentManager`. -(backported from #9269) From 48c4e1bbeb843df81d97919b56676888fd971f76 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 24 Aug 2021 17:46:50 +0000 Subject: [PATCH 53/79] Release 3.14.5 Redmine Query: https://pulp.plan.io/issues?set_filter=1&status_id=*&issue_id=9244,9282,9261,9268,9264,9273,9265 Redmine Milestone: https://pulp.plan.io/versions/278.json [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1d18b429bc..2a1e0fabd4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.5.dev +current_version = 3.14.5 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 2b5091a792..a7fdbf0bc4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.5.dev" +version = "3.14.5" # The full version, including alpha/beta/rc tags. -release = "3.14.5.dev" +release = "3.14.5" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 0903fa53ef..6b2af6e4c8 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.5.dev" + version = "3.14.5" def ready(self): super().ready() diff --git a/setup.py b/setup.py index dcac6fc1d5..b38c2dec35 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.5.dev", + version="3.14.5", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 371c5ec68272640608116f173b6bda319995396c Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 24 Aug 2021 17:46:50 +0000 Subject: [PATCH 54/79] Bump to 3.14.6.dev [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2a1e0fabd4..51b46750af 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.5 +current_version = 3.14.6.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index a7fdbf0bc4..5968e2d72a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.5" +version = "3.14.6.dev" # The full version, including alpha/beta/rc tags. -release = "3.14.5" +release = "3.14.6.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 6b2af6e4c8..54c8899cc7 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.5" + version = "3.14.6.dev" def ready(self): super().ready() diff --git a/setup.py b/setup.py index b38c2dec35..4889ec681a 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.5", + version="3.14.6.dev", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From b995a66c61e4f4c922bab24dbcb3fec6e735336a Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Wed, 25 Aug 2021 13:35:27 +0200 Subject: [PATCH 55/79] Stop using blake2s to calulate lock identifiers Instead perform a simple xor on the first and second half of the task uuid. backports #9249 fixes #9288 (cherry picked from commit 1d77e5cdf4d03b7edff34f4fc3b60a2f70df3e88) --- CHANGES/9288.bugfix | 2 ++ pulpcore/app/models/task.py | 8 +++----- pulpcore/tasking/util.py | 6 ------ 3 files changed, 5 insertions(+), 11 deletions(-) create mode 100644 CHANGES/9288.bugfix diff --git a/CHANGES/9288.bugfix b/CHANGES/9288.bugfix new file mode 100644 index 0000000000..799895f7f1 --- /dev/null +++ b/CHANGES/9288.bugfix @@ -0,0 +1,2 @@ +Stop using insecure hash function blake2s for calculating 64 bit lock identifier from uuid. +(backported from #9249) diff --git a/pulpcore/app/models/task.py b/pulpcore/app/models/task.py index 4ef5e5bf6e..27b6efb5c8 100644 --- a/pulpcore/app/models/task.py +++ b/pulpcore/app/models/task.py @@ -7,7 +7,6 @@ from contextlib import suppress from datetime import timedelta from gettext import gettext as _ -from hashlib import blake2s from django.contrib.postgres.fields import JSONField, ArrayField from django.db import connection, models @@ -209,9 +208,8 @@ def save_heartbeat(self): self.save(update_fields=["last_heartbeat"]) -def _hash_to_u64(value): - _digest = blake2s(value.encode(), digest_size=8).digest() - return int.from_bytes(_digest, byteorder="big", signed=True) +def _uuid_to_advisory_lock(value): + return ((value >> 64) ^ value) & 0x7FFFFFFFFFFFFFFF class TaskManager(models.Manager): @@ -286,7 +284,7 @@ def __str__(self): return "Task: {name} [{state}]".format(name=self.name, state=self.state) def __enter__(self): - self.lock = _hash_to_u64(str(self.pk)) + self.lock = _uuid_to_advisory_lock(self.pk.int) with connection.cursor() as cursor: cursor.execute("SELECT pg_try_advisory_lock(%s);", [self.lock]) acquired = cursor.fetchone()[0] diff --git a/pulpcore/tasking/util.py b/pulpcore/tasking/util.py index b985a6bc5b..d85387be23 100644 --- a/pulpcore/tasking/util.py +++ b/pulpcore/tasking/util.py @@ -1,6 +1,5 @@ import logging import time -from hashlib import blake2s from gettext import gettext as _ from django.conf import settings @@ -140,8 +139,3 @@ def get_current_worker(): return worker return None - - -def _hash_to_u64(value): - _digest = blake2s(value.encode(), digest_size=8).digest() - return int.from_bytes(_digest, byteorder="big", signed=True) From b9652e8330fc89d4083b3f07bc2f764dbb2812ce Mon Sep 17 00:00:00 2001 From: Brian Bouterse Date: Thu, 2 Sep 2021 13:15:08 -0400 Subject: [PATCH 56/79] Don't send Content-Encoding header with streamed responses aiohttp automatically enflates gzipped responses. Pulp clients always receive uncompressed responses when requesting on_demand content. backports: #9213 fixes #9325 (cherry picked from commit 7e0a79e4ea37a0e6694896dca35d126125ec5bdd) --- CHANGES/9325.bugfix | 3 +++ pulpcore/content/handler.py | 1 + 2 files changed, 4 insertions(+) create mode 100644 CHANGES/9325.bugfix diff --git a/CHANGES/9325.bugfix b/CHANGES/9325.bugfix new file mode 100644 index 0000000000..8e0b18f25b --- /dev/null +++ b/CHANGES/9325.bugfix @@ -0,0 +1,3 @@ +Fixed a bug where ``pulpcore-content`` decompressed data while incorrectly advertising to clients +it was still compressed via the ``Content-Encoding: gzip`` header. +(backported from #9213) diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py index 7c9a1e3dad..1295c31282 100644 --- a/pulpcore/content/handler.py +++ b/pulpcore/content/handler.py @@ -101,6 +101,7 @@ class Handler: hop_by_hop_headers = [ "connection", + "content-encoding", "keep-alive", "public", "proxy-authenticate", From f4fe7276ea5af5afb6c65157814fcee24750f162 Mon Sep 17 00:00:00 2001 From: Brian Bouterse Date: Thu, 2 Sep 2021 13:00:24 -0400 Subject: [PATCH 57/79] backports worker not starting on openshift For some strange reason some Openshift environments do not like the runtime import the worker entrypoint used. Moving it to the top where it belongs anyway resolves the problem. backports #9338 fixes #9339 (cherry picked from commit 022d2e390729401e157201edea84da14c77b49db) --- CHANGES/9339.bugfix | 3 +++ pulpcore/tasking/entrypoint.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 CHANGES/9339.bugfix diff --git a/CHANGES/9339.bugfix b/CHANGES/9339.bugfix new file mode 100644 index 0000000000..b1313f5655 --- /dev/null +++ b/CHANGES/9339.bugfix @@ -0,0 +1,3 @@ +Fixed bug where some Openshift environments could not start workers due to a strange Python runtime +import issue. +(backported from #9338) diff --git a/pulpcore/tasking/entrypoint.py b/pulpcore/tasking/entrypoint.py index e2d76a4487..22dbddf4ab 100644 --- a/pulpcore/tasking/entrypoint.py +++ b/pulpcore/tasking/entrypoint.py @@ -14,6 +14,7 @@ django.setup() from django.conf import settings # noqa: E402: module level not at top of file +from pulpcore.tasking.pulpcore_worker import NewPulpWorker # noqa: E402: module level not at top _logger = logging.getLogger(__name__) @@ -40,8 +41,6 @@ def worker(resource_manager, pid): ) select.select([], [], []) _logger.info("Starting distributed type worker") - from pulpcore.tasking.pulpcore_worker import NewPulpWorker - NewPulpWorker().run_forever() else: _logger.info("Starting rq type worker") From 5ac90c429ff336791cde1162c695d1413eab3fed Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 2 Sep 2021 17:48:15 +0000 Subject: [PATCH 58/79] Building changelog for 3.14.6 [noissue] --- CHANGES.rst | 27 +++++++++++++++++++++++++++ CHANGES/9288.bugfix | 2 -- CHANGES/9325.bugfix | 3 --- CHANGES/9339.bugfix | 3 --- 4 files changed, 27 insertions(+), 8 deletions(-) delete mode 100644 CHANGES/9288.bugfix delete mode 100644 CHANGES/9325.bugfix delete mode 100644 CHANGES/9339.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index 5bd7f304b3..946b4c8a62 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,33 @@ Changelog .. towncrier release notes start +3.14.6 (2021-09-02) +=================== +REST API +-------- + +Bugfixes +~~~~~~~~ + +- Stop using insecure hash function blake2s for calculating 64 bit lock identifier from uuid. + (backported from #9249) + `#9288 `_ +- Fixed a bug where ``pulpcore-content`` decompressed data while incorrectly advertising to clients + it was still compressed via the ``Content-Encoding: gzip`` header. + (backported from #9213) + `#9325 `_ +- Fixed bug where some Openshift environments could not start workers due to a strange Python runtime + import issue. + (backported from #9338) + `#9339 `_ + + +Plugin API +---------- + +No significant changes. + + 3.14.5 (2021-08-24) =================== REST API diff --git a/CHANGES/9288.bugfix b/CHANGES/9288.bugfix deleted file mode 100644 index 799895f7f1..0000000000 --- a/CHANGES/9288.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Stop using insecure hash function blake2s for calculating 64 bit lock identifier from uuid. -(backported from #9249) diff --git a/CHANGES/9325.bugfix b/CHANGES/9325.bugfix deleted file mode 100644 index 8e0b18f25b..0000000000 --- a/CHANGES/9325.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug where ``pulpcore-content`` decompressed data while incorrectly advertising to clients -it was still compressed via the ``Content-Encoding: gzip`` header. -(backported from #9213) diff --git a/CHANGES/9339.bugfix b/CHANGES/9339.bugfix deleted file mode 100644 index b1313f5655..0000000000 --- a/CHANGES/9339.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Fixed bug where some Openshift environments could not start workers due to a strange Python runtime -import issue. -(backported from #9338) From df93ea9df77003b38352b18d35fa6d37a0f76e67 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 2 Sep 2021 17:48:17 +0000 Subject: [PATCH 59/79] Release 3.14.6 Redmine Query: https://pulp.plan.io/issues?set_filter=1&status_id=*&issue_id=9288,9325,9339 Redmine Milestone: https://pulp.plan.io/versions/281.json [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 51b46750af..cb1293fc80 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.6.dev +current_version = 3.14.6 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 5968e2d72a..8c1cb0245a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.6.dev" +version = "3.14.6" # The full version, including alpha/beta/rc tags. -release = "3.14.6.dev" +release = "3.14.6" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 54c8899cc7..af861933d4 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.6.dev" + version = "3.14.6" def ready(self): super().ready() diff --git a/setup.py b/setup.py index 4889ec681a..945802c61e 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.6.dev", + version="3.14.6", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From dca870eb00fa1986cb4c59849f3fe4b2c2751b51 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 2 Sep 2021 17:48:17 +0000 Subject: [PATCH 60/79] Bump to 3.14.7.dev [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cb1293fc80..920ec51e56 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.6 +current_version = 3.14.7.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 8c1cb0245a..56a2a0f0bd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.6" +version = "3.14.7.dev" # The full version, including alpha/beta/rc tags. -release = "3.14.6" +release = "3.14.7.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index af861933d4..77976dee82 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.6" + version = "3.14.7.dev" def ready(self): super().ready() diff --git a/setup.py b/setup.py index 945802c61e..82a892dc8f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.6", + version="3.14.7.dev", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 1f7ec2409617200ee7e913cfb21046efce9b1529 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Wed, 22 Sep 2021 09:17:40 -0400 Subject: [PATCH 61/79] Do bulk_touch instead of touching one by one backports #9266. closes: #9401 (cherry picked from commit 4023fce) --- CHANGES/9401.bugfix | 2 ++ pulpcore/plugin/actions.py | 53 ++++++++++++++++++++++++++------------ 2 files changed, 38 insertions(+), 17 deletions(-) create mode 100644 CHANGES/9401.bugfix diff --git a/CHANGES/9401.bugfix b/CHANGES/9401.bugfix new file mode 100644 index 0000000000..ffe27bd128 --- /dev/null +++ b/CHANGES/9401.bugfix @@ -0,0 +1,2 @@ +Fixed the repository modify endpoint performance problems. +(backported from #9266) diff --git a/pulpcore/plugin/actions.py b/pulpcore/plugin/actions.py index 25a7299111..418af8b0c7 100644 --- a/pulpcore/plugin/actions.py +++ b/pulpcore/plugin/actions.py @@ -1,6 +1,7 @@ -from django.db import DatabaseError +from gettext import gettext as _ from drf_spectacular.utils import extend_schema from rest_framework.decorators import action +from rest_framework.serializers import ValidationError from pulpcore.app import tasks from pulpcore.app.models import Content, RepositoryVersion @@ -9,6 +10,7 @@ AsyncOperationResponseSerializer, RepositoryAddRemoveContentSerializer, ) +from pulpcore.app.viewsets import NamedModelViewSet from pulpcore.tasking.tasks import dispatch @@ -26,8 +28,9 @@ def modify(self, request, pk): """ Queues a task that creates a new RepositoryVersion by adding and removing content units """ - add_content_units = [] - remove_content_units = [] + add_content_units = {} + remove_content_units = {} + repository = self.get_object() serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) @@ -39,22 +42,26 @@ def modify(self, request, pk): if "add_content_units" in request.data: for url in request.data["add_content_units"]: - content = self.get_resource(url, Content) - try: - content.touch() - except DatabaseError: - # content has since been removed. call get_url to raise an exception. - content = self.get_resource(url, Content) - add_content_units.append(content.pk) + add_content_units[NamedModelViewSet.extract_pk(url)] = url + + content_units_pks = set(add_content_units.keys()) + existing_content_units = Content.objects.filter(pk__in=content_units_pks) + existing_content_units.touch() + + self.verify_content_units(existing_content_units, add_content_units) + + add_content_units = list(add_content_units.keys()) if "remove_content_units" in request.data: - for url in request.data["remove_content_units"]: - if url == "*": - remove_content_units = [url] - break - else: - content = self.get_resource(url, Content) - remove_content_units.append(content.pk) + if "*" in request.data["remove_content_units"]: + remove_content_units = ["*"] + else: + for url in request.data["remove_content_units"]: + remove_content_units[NamedModelViewSet.extract_pk(url)] = url + content_units_pks = set(remove_content_units.keys()) + existing_content_units = Content.objects.filter(pk__in=content_units_pks) + self.verify_content_units(existing_content_units, remove_content_units) + remove_content_units = list(remove_content_units.keys()) task = dispatch( tasks.repository.add_and_remove, @@ -67,3 +74,15 @@ def modify(self, request, pk): }, ) return OperationPostponedResponse(task, request) + + def verify_content_units(self, content_units, all_content_units): + """Verify referenced content units.""" + existing_content_units_pks = content_units.values_list("pk", flat=True) + existing_content_units_pks = {str(pk) for pk in existing_content_units_pks} + + missing_pks = set(all_content_units.keys()) - existing_content_units_pks + if missing_pks: + missing_hrefs = [all_content_units[pk] for pk in missing_pks] + raise ValidationError( + _("Could not find the following content units: {}").format(missing_hrefs) + ) From d618a13c1cbb9c16ae1d91acf972576858f0ae1a Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Wed, 22 Sep 2021 09:36:18 -0400 Subject: [PATCH 62/79] Update remote artifact urls on sync if the remote or repo changes backports #9395. closes: #9400 (cherrypicked from commit 489156e) --- CHANGES/9400.bugfix | 2 + pulpcore/plugin/stages/artifact_stages.py | 24 ++++-- .../functional/api/using_plugin/constants.py | 14 ++++ .../api/using_plugin/test_content_delivery.py | 79 ++++++++++++++++++- 4 files changed, 111 insertions(+), 8 deletions(-) create mode 100644 CHANGES/9400.bugfix diff --git a/CHANGES/9400.bugfix b/CHANGES/9400.bugfix new file mode 100644 index 0000000000..0fa363ee39 --- /dev/null +++ b/CHANGES/9400.bugfix @@ -0,0 +1,2 @@ +Fixed an issue where on_demand content might not be downloaded properly if the remote URL was changed (even if re-synced). +(backported from #9395) diff --git a/pulpcore/plugin/stages/artifact_stages.py b/pulpcore/plugin/stages/artifact_stages.py index 9eda6d1d69..66d7416f2b 100644 --- a/pulpcore/plugin/stages/artifact_stages.py +++ b/pulpcore/plugin/stages/artifact_stages.py @@ -267,11 +267,11 @@ async def run(self): The coroutine for this stage. """ async for batch in self.batches(): - RemoteArtifact.objects.bulk_get_or_create(self._needed_remote_artifacts(batch)) + self._handle_remote_artifacts(batch) for d_content in batch: await self.put(d_content) - def _needed_remote_artifacts(self, batch): + def _handle_remote_artifacts(self, batch): """ Build a list of only :class:`~pulpcore.plugin.models.RemoteArtifact` that need to be created for the batch. @@ -313,7 +313,8 @@ def _needed_remote_artifacts(self, batch): # # We can end up with duplicates (diff pks, same sha256) in the sequence below, # so we store by-sha256 and then return the final values - needed_ras = {} # { str(): RemoteArtifact, ... } + ras_to_create = {} # { str(): RemoteArtifact, ... } + ras_to_update = {} for d_content in batch: for d_artifact in d_content.d_artifacts: if not d_artifact.remote: @@ -370,14 +371,23 @@ def _needed_remote_artifacts(self, batch): ) for remote_artifact in content_artifact._remote_artifact_saver_ras: - if remote_artifact.remote_id == d_artifact.remote.pk: + if d_artifact.url == remote_artifact.url: + break + + if d_artifact.remote.pk == remote_artifact.remote_id: + key = f"{content_artifact.pk}-{remote_artifact.remote_id}" + remote_artifact.url = d_artifact.url + ras_to_update[key] = remote_artifact break else: remote_artifact = self._create_remote_artifact(d_artifact, content_artifact) - key = f"{str(content_artifact.pk)}-{str(d_artifact.remote.pk)}" - needed_ras[key] = remote_artifact + key = f"{content_artifact.pk}-{d_artifact.remote.pk}" + ras_to_create[key] = remote_artifact - return list(needed_ras.values()) + if ras_to_create: + RemoteArtifact.objects.bulk_create(list(ras_to_create.values())) + if ras_to_update: + RemoteArtifact.objects.bulk_update(list(ras_to_update.values()), fields=["url"]) @staticmethod def _create_remote_artifact(d_artifact, content_artifact): diff --git a/pulpcore/tests/functional/api/using_plugin/constants.py b/pulpcore/tests/functional/api/using_plugin/constants.py index 4cbc3928a2..5ccd93d5e6 100644 --- a/pulpcore/tests/functional/api/using_plugin/constants.py +++ b/pulpcore/tests/functional/api/using_plugin/constants.py @@ -47,6 +47,20 @@ FILE_FIXTURE_URL = urljoin(PULP_FIXTURES_BASE_URL, "file/") """The URL to a file repository.""" +FILE_FIXTURE_WITH_MISSING_FILES_URL = urljoin(PULP_FIXTURES_BASE_URL, "file-manifest/") +"""The URL to a file repository with missing files.""" + +FILE_FIXTURE_WITH_MISSING_FILES_MANIFEST_URL = urljoin( + FILE_FIXTURE_WITH_MISSING_FILES_URL, "PULP_MANIFEST" +) +"""The URL to a file repository with missing files manifest.""" + +FILE_CHUNKED_FIXTURE_URL = urljoin(PULP_FIXTURES_BASE_URL, "file-chunked/") +"""The URL to a file repository.""" + +FILE_CHUNKED_FIXTURE_MANIFEST_URL = urljoin(FILE_CHUNKED_FIXTURE_URL, "PULP_MANIFEST") +"""The URL to a file repository manifest""" + FILE_FIXTURE_MANIFEST_URL = urljoin(FILE_FIXTURE_URL, "PULP_MANIFEST") """The URL to a file repository manifest.""" diff --git a/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py b/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py index 116c7164fc..5e1b7ac904 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py +++ b/pulpcore/tests/functional/api/using_plugin/test_content_delivery.py @@ -5,7 +5,7 @@ from urllib.parse import urljoin from pulp_smash import api, config, utils -from pulp_smash.pulp3.bindings import delete_orphans +from pulp_smash.pulp3.bindings import delete_orphans, monitor_task, PulpTestCase from pulp_smash.pulp3.constants import ON_DEMAND_DOWNLOAD_POLICIES from pulp_smash.pulp3.utils import ( download_content_unit, @@ -16,16 +16,26 @@ ) from requests import HTTPError +from pulpcore.client.pulp_file import ( + PublicationsFileApi, + RemotesFileApi, + RepositoriesFileApi, + RepositorySyncURL, + DistributionsFileApi, +) from pulpcore.tests.functional.api.using_plugin.constants import ( FILE_CONTENT_NAME, FILE_DISTRIBUTION_PATH, FILE_FIXTURE_URL, + FILE_FIXTURE_MANIFEST_URL, + FILE_FIXTURE_WITH_MISSING_FILES_MANIFEST_URL, FILE_REMOTE_PATH, FILE_REPO_PATH, ) from pulpcore.tests.functional.api.using_plugin.utils import ( create_file_publication, gen_file_remote, + gen_file_client, ) from pulpcore.tests.functional.api.using_plugin.utils import ( # noqa:F401 set_up_module as setUpModule, @@ -105,3 +115,70 @@ def test_content_remote_delete(self): ).hexdigest() self.assertEqual(pulp_hash, fixtures_hash) + + +class RemoteArtifactUpdateTestCase(PulpTestCase): + @classmethod + def setUpClass(cls): + """Clean out Pulp before testing.""" + delete_orphans() + client = gen_file_client() + cls.repo_api = RepositoriesFileApi(client) + cls.remote_api = RemotesFileApi(client) + cls.publication_api = PublicationsFileApi(client) + cls.distributions_api = DistributionsFileApi(client) + cls.cfg = config.get_config() + + def tearDown(self): + """Clean up Pulp after testing.""" + self.doCleanups() + delete_orphans() + + def test_remote_artifact_url_update(self): + """Test that downloading on_demand content works after a repository layout change.""" + + FILE_NAME = "1.iso" + + # 1. Create a remote, repository and distribution - remote URL has links that should 404 + remote_config = gen_file_remote( + policy="on_demand", url=FILE_FIXTURE_WITH_MISSING_FILES_MANIFEST_URL + ) + remote = self.remote_api.create(remote_config) + self.addCleanup(self.remote_api.delete, remote.pulp_href) + + repo = self.repo_api.create(gen_repo(autopublish=True, remote=remote.pulp_href)) + self.addCleanup(self.repo_api.delete, repo.pulp_href) + + body = gen_distribution(repository=repo.pulp_href) + distribution_response = self.distributions_api.create(body) + created_resources = monitor_task(distribution_response.task).created_resources + distribution = self.distributions_api.read(created_resources[0]) + self.addCleanup(self.distributions_api.delete, distribution.pulp_href) + + # 2. Sync the repository, verify that downloading artifacts fails + repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + + sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data) + monitor_task(sync_response.task) + + with self.assertRaises(HTTPError): + download_content_unit(self.cfg, distribution.to_dict(), FILE_NAME) + + # 3. Update the remote URL with one that works, sync again, check that downloading + # artifacts works. + update_response = self.remote_api.update( + remote.pulp_href, gen_file_remote(policy="on_demand", url=FILE_FIXTURE_MANIFEST_URL) + ) + monitor_task(update_response.task) + + sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data) + monitor_task(sync_response.task) + + content = download_content_unit(self.cfg, distribution.to_dict(), FILE_NAME) + pulp_hash = hashlib.sha256(content).hexdigest() + + fixtures_hash = hashlib.sha256( + utils.http_get(urljoin(FILE_FIXTURE_URL, FILE_NAME)) + ).hexdigest() + + self.assertEqual(pulp_hash, fixtures_hash) From 9f6be1f4263a288ef03be5e398f62c3e1c911695 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Thu, 23 Sep 2021 15:03:49 -0400 Subject: [PATCH 63/79] Replaced incorrect use of 'filename' with 'natural_keys()' in error log. backports #9427. closes: #9440 [nocoverage] (cherry picked from commit 6e18f2e75cbcbb673ede4bd102171ef4dfc4b860) --- CHANGES/9440.bugfix | 2 ++ pulpcore/plugin/stages/artifact_stages.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 CHANGES/9440.bugfix diff --git a/CHANGES/9440.bugfix b/CHANGES/9440.bugfix new file mode 100644 index 0000000000..8a386c683b --- /dev/null +++ b/CHANGES/9440.bugfix @@ -0,0 +1,2 @@ +Taught a remote-artifact error path to not assume 'filename' was valid for all content. +(backported from #9427) diff --git a/pulpcore/plugin/stages/artifact_stages.py b/pulpcore/plugin/stages/artifact_stages.py index 66d7416f2b..950045c70b 100644 --- a/pulpcore/plugin/stages/artifact_stages.py +++ b/pulpcore/plugin/stages/artifact_stages.py @@ -346,7 +346,7 @@ def _handle_remote_artifacts(self, batch): log.warning( msg.format( rp=d_artifact.relative_path, - c=d_content.content.filename, + c=d_content.content.natural_key(), rname=d_artifact.remote.name, ap=avail_paths, ) @@ -360,7 +360,7 @@ def _handle_remote_artifacts(self, batch): raise ValueError( msg.format( rp=d_artifact.relative_path, - c=d_content.content.filename, + c=d_content.content.natural_key(), rname=d_artifact.remote.name, ) ) From 2f3395d3976381ffbc1c923278cd2c73269696e5 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Mon, 27 Sep 2021 13:30:13 -0400 Subject: [PATCH 64/79] Update CI [noissue] --- .ci/ansible/Containerfile.j2 | 7 +- .ci/ansible/settings.py.j2 | 13 ++ .github/template_gitref | 2 +- .github/workflows/ci.yml | 140 ++++++++++++++++-- .github/workflows/create-branch.yml | 3 + .github/workflows/nightly.yml | 102 +++++++++++-- .github/workflows/release.yml | 62 +++++++- .github/workflows/scripts/before_install.sh | 23 +-- .github/workflows/scripts/before_script.sh | 2 +- .github/workflows/scripts/check_commit.sh | 2 +- .github/workflows/scripts/install.sh | 7 +- .github/workflows/scripts/release.py | 2 +- .github/workflows/scripts/script.sh | 6 +- .../scripts/stage-changelog-for-master.py | 2 +- .github/workflows/scripts/update_ci.sh | 7 + .github/workflows/update_ci.yml | 11 +- template_config.yml | 13 +- 17 files changed, 347 insertions(+), 57 deletions(-) diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index de47e4dac8..4483eab8a0 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -14,6 +14,9 @@ RUN pip3 install \ {%- if s3_test | default(false) -%} {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue {%- endif -%} +{%- if azure_test | default(false) -%} +{{ " " }}django-storages[azure] +{%- endif -%} {%- for item in plugins -%} {%- if item.name == "pulp-certguard" -%} {{ " " }}python-dateutil rhsm @@ -23,8 +26,8 @@ RUN pip3 install \ RUN mkdir -p /etc/nginx/pulp/ {% for item in plugins %} -RUN export plugin_path="$(pip show {{ item.name }} | sed -n -e "s/Location: //p")/{{ item.name }}" -RUN ln "$plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf" || true +RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ + ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true {% endfor %} ENTRYPOINT ["/init"] diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index e0e040a7ac..9e62d935bb 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -23,4 +23,17 @@ AWS_STORAGE_BUCKET_NAME = "pulp3" AWS_S3_ENDPOINT_URL = "http://minio:9000" DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" AWS_DEFAULT_ACL = "@none None" +MEDIA_ROOT = "" +{% endif %} + +{% if azure_test | default(false) %} +AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" +AZURE_ACCOUNT_NAME = "devstoreaccount1" +AZURE_CONTAINER = "pulp-test" +AZURE_LOCATION = "pulp3" +AZURE_OVERWRITE_FILES = True +AZURE_URL_EXPIRATION_SECS = 120 +AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://pulp-azurite:10000/devstoreaccount1;' +DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" +MEDIA_ROOT = "" {% endif %} diff --git a/.github/template_gitref b/.github/template_gitref index 4bfaa589fc..9743715921 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.04.08-84-g16a91e9-dirty +2021.04.08-122-gefc11dd diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46d42ff4a1..94ef2212bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,17 +4,32 @@ # './plugin-template --github pulpcore' to update this file. # # For more info visit https://github.com/pulp/plugin_template ---- -name: Pulp CI -on: - pull_request: - branches: - - '*' +--- +name: Pulpcore CI +on: {pull_request: {branches: ['*']}} jobs: + + single_commit: + name: Assert single commit + if: github.base_ref == 'master' + steps: + - name: Checkout + uses: actions/checkout@v1 + with: + fetch-depth: 30 + - name: Checkout master + run: git fetch origin master + - name: create local master branch + run: git branch master origin/master + - name: Commit Count Check + run: test `git log --oneline --no-merges HEAD ^master | wc -l ` = 1 + runs-on: ubuntu-latest + lint: runs-on: ubuntu-latest + steps: - uses: actions/checkout@v2 @@ -34,6 +49,13 @@ jobs: - name: Check commit message if: github.event_name == 'pull_request' env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} run: sh .github/workflows/scripts/check_commit.sh @@ -80,6 +102,7 @@ jobs: with: python-version: "3.6" - uses: actions/setup-ruby@v1 + with: ruby-version: "2.6" @@ -89,30 +112,63 @@ jobs: sudo apt-get update -yq sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV + echo "FROM_PULP_CERTGUARD_BRANCH=${{ matrix.FROM_PULP_CERTGUARD_BRANCH }}" >> $GITHUB_ENV + echo "FROM_PULP_FILE_BRANCH=${{ matrix.FROM_PULP_FILE_BRANCH }}" >> $GITHUB_ENV + echo "FROM_PULPCORE_BRANCH=${{ matrix.FROM_PULPCORE_BRANCH }}" >> $GITHUB_ENV - name: Before Install + run: .github/workflows/scripts/before_install.sh shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Install + run: .github/workflows/scripts/install.sh + shell: bash env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' - shell: bash + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + shell: bash - name: Install Ruby client if: ${{ env.TEST == 'bindings' }} run: .github/workflows/scripts/install_ruby_client.sh + shell: bash - name: Before Script - run: | - .github/workflows/scripts/before_script.sh + + run: .github/workflows/scripts/before_script.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Setting secrets if: github.event_name != 'pull_request' @@ -121,8 +177,18 @@ jobs: SECRETS_CONTEXT: ${{ toJson(secrets) }} - name: Script + run: .github/workflows/scripts/script.sh shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: After failure if: failure() @@ -142,6 +208,8 @@ jobs: strategy: fail-fast: false matrix: + env: + - TEST: upgrade include: - FROM_PULPCORE_BRANCH: "3.11" FROM_PULP_CERTGUARD_BRANCH: "1.2" @@ -164,7 +232,7 @@ jobs: sudo apt-get update -yq sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie echo ::endgroup:: - echo "TEST=upgrade" >> $GITHUB_ENV + echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV echo "FROM_PULP_CERTGUARD_BRANCH=${{ matrix.FROM_PULP_CERTGUARD_BRANCH }}" >> $GITHUB_ENV echo "FROM_PULP_FILE_BRANCH=${{ matrix.FROM_PULP_FILE_BRANCH }}" >> $GITHUB_ENV @@ -177,33 +245,76 @@ jobs: echo ::endgroup:: - name: Before Install + run: .github/workflows/scripts/before_install.sh shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Install + run: .github/workflows/scripts/install.sh + shell: bash env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' - shell: bash + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + shell: bash - name: Before Script + run: .github/workflows/scripts/before_script.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: SECRETS_CONTEXT: ${{ toJson(secrets) }} - - name: Upgrade test + - name: Upgrade Test + run: .github/workflows/scripts/script.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: After failure if: failure() run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true docker images || true docker ps -a || true @@ -211,3 +322,6 @@ jobs: docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true docker exec pulp pip3 list + + + diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 8b2a7c772d..8eb970ed9c 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -4,6 +4,7 @@ # './plugin-template --github pulpcore' to update this file. # # For more info visit https://github.com/pulp/plugin_template + --- name: Create New Release Branch on: @@ -41,6 +42,7 @@ jobs: echo ::endgroup:: - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: SECRETS_CONTEXT: ${{ toJson(secrets) }} @@ -67,6 +69,7 @@ jobs: - name: Make a PR with version bump uses: peter-evans/create-pull-request@v3 with: + token: ${{ secrets.RELEASE_TOKEN }} committer: pulpbot author: pulpbot branch: minor-version-bump diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b3de58e0b6..824285d79b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -4,8 +4,9 @@ # './plugin-template --github pulpcore' to update this file. # # For more info visit https://github.com/pulp/plugin_template + --- -name: Pulp Nightly CI/CD +name: Pulpcore Nightly CI/CD on: schedule: # * is a special character in YAML so you have to quote this string @@ -42,44 +43,88 @@ jobs: sudo apt-get update -yq sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV + echo "FROM_PULP_CERTGUARD_BRANCH=${{ matrix.FROM_PULP_CERTGUARD_BRANCH }}" >> $GITHUB_ENV + echo "FROM_PULP_FILE_BRANCH=${{ matrix.FROM_PULP_FILE_BRANCH }}" >> $GITHUB_ENV + echo "FROM_PULPCORE_BRANCH=${{ matrix.FROM_PULPCORE_BRANCH }}" >> $GITHUB_ENV - name: Before Install + run: .github/workflows/scripts/before_install.sh shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - uses: ruby/setup-ruby@v1 + - uses: actions/setup-ruby@v1 if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} with: ruby-version: "2.6" - name: Install + run: .github/workflows/scripts/install.sh + shell: bash env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' - shell: bash + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Before Script - run: | - .github/workflows/scripts/before_script.sh + + run: .github/workflows/scripts/before_script.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: SECRETS_CONTEXT: ${{ toJson(secrets) }} - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + shell: bash - name: Install Ruby client if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} run: .github/workflows/scripts/install_ruby_client.sh + shell: bash - name: Script + run: .github/workflows/scripts/script.sh shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Upload python client packages if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} @@ -105,6 +150,7 @@ jobs: - name: After failure if: failure() run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true docker images || true docker ps -a || true @@ -132,6 +178,7 @@ jobs: python-version: "3.6" - uses: actions/setup-ruby@v1 + with: ruby-version: "2.6" @@ -141,7 +188,11 @@ jobs: sudo apt-get update -yq sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie echo ::endgroup:: + echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV + echo "FROM_PULP_CERTGUARD_BRANCH=${{ matrix.FROM_PULP_CERTGUARD_BRANCH }}" >> $GITHUB_ENV + echo "FROM_PULP_FILE_BRANCH=${{ matrix.FROM_PULP_FILE_BRANCH }}" >> $GITHUB_ENV + echo "FROM_PULPCORE_BRANCH=${{ matrix.FROM_PULPCORE_BRANCH }}" >> $GITHUB_ENV - name: Install python dependencies run: | @@ -150,27 +201,59 @@ jobs: echo ::endgroup:: - name: Before Install + run: .github/workflows/scripts/before_install.sh shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Install + run: .github/workflows/scripts/install.sh + shell: bash env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' - shell: bash + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + shell: bash - name: Install Ruby client - if: ${{ env.TEST == 'bindings' }} || env.TEST == 'generate-bindings' }} + run: .github/workflows/scripts/install_ruby_client.sh + shell: bash - name: Before Script + run: .github/workflows/scripts/before_script.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: SECRETS_CONTEXT: ${{ toJson(secrets) }} @@ -179,7 +262,7 @@ jobs: - name: Download Ruby client uses: actions/download-artifact@v2 with: - name: python-client.tar + name: ruby-client.tar - name: Untar Ruby client packages run: tar -xvf ruby-client.tar @@ -216,6 +299,7 @@ jobs: - name: After failure if: failure() run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true docker images || true docker ps -a || true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe2c8a215f..0bcde5ac27 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,7 @@ # './plugin-template --github pulpcore' to update this file. # # For more info visit https://github.com/pulp/plugin_template + --- name: Release Pipeline on: @@ -51,6 +52,7 @@ jobs: git config --global user.email 'pulp-infra@redhat.com' - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: SECRETS_CONTEXT: ${{ toJson(secrets) }} @@ -89,6 +91,7 @@ jobs: with: python-version: "3.6" - uses: actions/setup-ruby@v1 + with: ruby-version: "2.6" @@ -112,12 +115,25 @@ jobs: sudo apt-get update -yq sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV + echo "FROM_PULP_CERTGUARD_BRANCH=${{ matrix.FROM_PULP_CERTGUARD_BRANCH }}" >> $GITHUB_ENV + echo "FROM_PULP_FILE_BRANCH=${{ matrix.FROM_PULP_FILE_BRANCH }}" >> $GITHUB_ENV + echo "FROM_PULPCORE_BRANCH=${{ matrix.FROM_PULPCORE_BRANCH }}" >> $GITHUB_ENV - name: Before Install + run: .github/workflows/scripts/before_install.sh shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Install run: | @@ -126,23 +142,43 @@ jobs: env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} shell: bash - name: Before Script - run: | - .github/workflows/scripts/before_script.sh + + run: .github/workflows/scripts/before_script.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: SECRETS_CONTEXT: ${{ toJson(secrets) }} - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + shell: bash - name: Install Ruby client if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} run: .github/workflows/scripts/install_ruby_client.sh + shell: bash - name: Additional before_script run: ${{ github.event.inputs.before_script }} @@ -152,6 +188,15 @@ jobs: if: ${{ env.TEST != 'generate-bindings' }} run: .github/workflows/scripts/script.sh shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - name: Upload python client packages if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} @@ -177,6 +222,7 @@ jobs: - name: After failure if: failure() run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true docker images || true docker ps -a || true @@ -204,6 +250,7 @@ jobs: python-version: "3.8" - uses: actions/setup-ruby@v1 + with: ruby-version: "2.6" @@ -227,6 +274,7 @@ jobs: git checkout "origin/${GITHUB_REF##*/}" -- .github - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: SECRETS_CONTEXT: ${{ toJson(secrets) }} @@ -286,13 +334,15 @@ jobs: - name: Create Pull Request for Changelog uses: peter-evans/create-pull-request@v3 with: + token: ${{ secrets.RELEASE_TOKEN }} committer: pulpbot author: pulpbot - branch: ${{ github.event.inputs.release }}-changelog + branch: changelog/${{ github.event.inputs.release }} base: master - title: 'Building changelog for ${{ github.event.inputs.release }}' + title: 'Cherry pick ${{ github.event.inputs.release }} changelog' body: '[noissue]' commit-message: | - Building changelog for ${{ github.event.inputs.release }} + ${{ github.event.inputs.release }} changelog + [noissue] delete-branch: true diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 1fd8b8986c..48bff9693d 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -31,6 +31,7 @@ COMMIT_MSG=$(git log --format=%B --no-merges -1) export COMMIT_MSG if [[ "$TEST" == "upgrade" ]]; then + pip install -r functest_requirements.txt git checkout -b ci_upgrade_test cp -R .github /tmp/.github cp -R .ci /tmp/.ci @@ -38,8 +39,6 @@ if [[ "$TEST" == "upgrade" ]]; then rm -rf .ci .github cp -R /tmp/.github . cp -R /tmp/.ci . - # Pin deps - sed -i "s/~/=/g" requirements.txt fi if [[ "$TEST" == "plugin-from-pypi" ]]; then @@ -117,36 +116,36 @@ cd .. git clone --depth=1 https://github.com/pulp/pulp_file.git --branch 1.8 +cd pulp_file + if [ -n "$PULP_FILE_PR_NUMBER" ]; then - cd pulp_file git fetch --depth=1 origin pull/$PULP_FILE_PR_NUMBER/head:$PULP_FILE_PR_NUMBER git checkout $PULP_FILE_PR_NUMBER - cd .. fi +cd .. + git clone --depth=1 https://github.com/pulp/pulp-certguard.git --branch 1.4 +cd pulp-certguard + if [ -n "$PULP_CERTGUARD_PR_NUMBER" ]; then - cd pulp-certguard git fetch --depth=1 origin pull/$PULP_CERTGUARD_PR_NUMBER/head:$PULP_CERTGUARD_PR_NUMBER git checkout $PULP_CERTGUARD_PR_NUMBER - cd .. fi +cd .. + if [[ "$TEST" == "upgrade" ]]; then cd pulp-certguard git checkout -b ci_upgrade_test git fetch --depth=1 origin heads/$FROM_PULP_CERTGUARD_BRANCH:$FROM_PULP_CERTGUARD_BRANCH git checkout $FROM_PULP_CERTGUARD_BRANCH - # Pin deps - sed -i "s/~/=/g" requirements.txt cd .. cd pulp_file git checkout -b ci_upgrade_test git fetch --depth=1 origin heads/$FROM_PULP_FILE_BRANCH:$FROM_PULP_FILE_BRANCH git checkout $FROM_PULP_FILE_BRANCH - # Pin deps - sed -i "s/~/=/g" requirements.txt cd .. fi @@ -156,7 +155,7 @@ pip install docker netaddr boto3 ansible for i in {1..3} do - ansible-galaxy collection install amazon.aws && s=0 && break || s=$? && sleep 3 + ansible-galaxy collection install "amazon.aws:1.5.0" && s=0 && break || s=$? && sleep 3 done if [[ $s -gt 0 ]] then @@ -164,6 +163,8 @@ then exit $s fi +sed -i -e 's/DEBUG = False/DEBUG = True/' pulpcore/pulpcore/app/settings.py + cd pulpcore if [ -f $POST_BEFORE_INSTALL ]; then diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 9af83da88e..3cce04b81e 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -29,7 +29,7 @@ tail -v -n +1 .ci/ansible/vars/main.yaml echo "PULP CONFIG:" tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json -if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == "plugin-from-pypi" ]]; then +if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == 'azure' || "$TEST" == "plugin-from-pypi" ]]; then # Many functional tests require these cmd_prefix dnf install -yq lsof which dnf-plugins-core fi diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index 095b7b44d3..61e897ceb0 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -17,7 +17,7 @@ pip3 install requests echo ::endgroup:: -for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') +for sha in $(curl -H "Authorization: token $GITHUB_TOKEN" $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') do python3 .ci/scripts/validate_commit_message.py $sha VALUE=$? diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index e0ecc2c392..07e8cba9b7 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -68,7 +68,7 @@ pulp_container_tag: python36 VARSYAML -if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "upgrade" || "$TEST" == "s3" || "$TEST" == "plugin-from-pypi" ]]; then +if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "upgrade" || "$TEST" == "azure" || "$TEST" == "s3" || "$TEST" == "plugin-from-pypi" ]]; then sed -i -e '/^services:/a \ - name: pulp-fixtures\ image: docker.io/pulp/pulp-fixtures:latest\ @@ -93,6 +93,11 @@ fi ansible-playbook build_container.yaml ansible-playbook start_container.yaml +if [ "$TEST" = "azure" ]; then + AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://pulp-azurite:10000/devstoreaccount1;' + az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING +fi + echo ::group::PIP_LIST cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" echo ::endgroup:: diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py index 8c6dc59bc0..3ace76393b 100755 --- a/.github/workflows/scripts/release.py +++ b/.github/workflows/scripts/release.py @@ -137,7 +137,7 @@ def create_release_commits(repo, release_version, plugin_path): git = repo.git git.add("CHANGES.rst") git.add("CHANGES/*") - git.commit("-m", f"Building changelog for {release_version}\n\n[noissue]") + git.commit("-m", f"{release_version} changelog\n\n[noissue]") # Second commit: release version os.system("bump2version release --allow-dirty") diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index c2ec615c25..752b9e013b 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -115,12 +115,12 @@ if [[ "$TEST" == "upgrade" ]]; then sed -i "/require_pulp_plugins(/d" pulpcore/tests/functional/utils.py # Running pre upgrade tests: - pytest -v -r sx --color=yes --pyargs -capture=no pulpcore.tests.upgrade.pre + pytest -v -r sx --color=yes --pyargs --capture=no pulpcore.tests.upgrade.pre # Checking out ci_upgrade_test branch and upgrading plugins + cmd_prefix bash -c "cd pulpcore; git checkout -f ci_upgrade_test; pip install --upgrade --force-reinstall ." cmd_prefix bash -c "cd pulp-certguard; git checkout -f ci_upgrade_test; pip install ." cmd_prefix bash -c "cd pulp_file; git checkout -f ci_upgrade_test; pip install ." - cmd_prefix bash -c "cd pulpcore; git checkout -f ci_upgrade_test; pip install ." # Migrating cmd_prefix bash -c "django-admin migrate --no-input" @@ -159,7 +159,7 @@ if [[ "$TEST" == "upgrade" ]]; then # Running post upgrade tests git checkout ci_upgrade_test -- pulpcore/tests/ - pytest -v -r sx --color=yes --pyargs -capture=no pulpcore.tests.upgrade.post + pytest -v -r sx --color=yes --pyargs --capture=no pulpcore.tests.upgrade.post exit fi diff --git a/.github/workflows/scripts/stage-changelog-for-master.py b/.github/workflows/scripts/stage-changelog-for-master.py index 0c344567ca..4a9c00ad71 100755 --- a/.github/workflows/scripts/stage-changelog-for-master.py +++ b/.github/workflows/scripts/stage-changelog-for-master.py @@ -44,7 +44,7 @@ changelog_commit = None # Look for a commit with the requested release version for commit in repo.iter_commits(): - if f"Building changelog for {release_version_arg}\n" in commit.message: + if f"{release_version_arg} changelog" == commit.message.split("\n")[0]: changelog_commit = commit break diff --git a/.github/workflows/scripts/update_ci.sh b/.github/workflows/scripts/update_ci.sh index b7fc256310..33f0bb98dc 100755 --- a/.github/workflows/scripts/update_ci.sh +++ b/.github/workflows/scripts/update_ci.sh @@ -17,6 +17,13 @@ pushd ../plugin_template ./plugin-template --github pulpcore popd +# Check if only gitref file has changed, so no effect on CI workflows. +if [[ `git diff --name-only` == ".github/template_gitref" ]]; then + echo "CI update has no effect on workflows, skipping PR creation." + git stash + exit 0 +fi + if [[ `git status --porcelain` ]]; then git add -A git commit -m "Update CI files" -m "[noissue]" diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index e0342596bd..92609bd4d6 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -9,8 +9,8 @@ name: CI Update on: schedule: # * is a special character in YAML so you have to quote this string - # runs at 2:30 UTC daily - - cron: '30 2 * * *' + # runs at 2:30 UTC every Sunday + - cron: '30 2 * * 0' workflow_dispatch: @@ -36,14 +36,13 @@ jobs: - name: Create Pull Request for CI files uses: peter-evans/create-pull-request@v3 with: + token: ${{ secrets.RELEASE_TOKEN }} committer: pulpbot author: pulpbot - branch: ${GITHUB_REF#refs/heads/}-update-ci-files - base: ${GITHUB_REF#refs/heads/} - title: 'Update CI for ${GITHUB_REF#refs/heads/}' + title: 'Update CI files' body: '[noissue]' commit-message: | - Update CI files for ${GITHUB_REF#refs/heads/} + Update CI files [noissue] delete-branch: true diff --git a/template_config.yml b/template_config.yml index 5ae3356cf2..e706848da0 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.04.08-84-g3271e88 +# generated with plugin_template@2021.04.08-122-gefc11dd additional_repos: - branch: 1.8 @@ -15,6 +15,9 @@ check_manifest: true check_openapi_schema: true check_stray_pulpcore_imports: false cherry_pick_automation: false +ci_env: {} +ci_trigger: '{pull_request: {branches: [''*'']}}' +core_import_allowed: [] coverage: true deploy_client_to_pypi: true deploy_client_to_rubygems: true @@ -25,6 +28,7 @@ docker_fixtures: true docs_test: true flake8: true issue_tracker: redmine +noissue_marker: '[noissue]' plugin_app_label: core plugin_camel: Pulpcore plugin_camel_short: Pulpcore @@ -35,6 +39,8 @@ plugin_dash_short: pulpcore plugin_default_branch: master plugin_name: pulpcore plugin_snake: pulpcore +post_job_template: null +pre_job_template: null publish_docs_to_pulpprojectdotorg: true pulp_scheme: http pulp_settings: @@ -44,14 +50,19 @@ pulp_settings: - /tmp pulpcore_branch: null pulpcore_pip_version_specifier: null +pulpcore_revision: null pulpprojectdotorg_key_id: d1a778bda808 pydocstyle: true pypi_username: pulp python_version: '3.6' redmine_project: pulp +release_email: pulp-infra@redhat.com release_user: pulpbot +single_commit_check: true stable_branch: null sync_ci: true +tasking_allow_async_unsafe: true +test_azure: false test_bindings: true test_cli: true test_fips_nightly: false From 69b2e1dcd9f2b51887d0faf856834fc820925f0d Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Tue, 28 Sep 2021 10:58:03 -0400 Subject: [PATCH 65/79] Teach the touch() path to apply order to avoid deadlocks. backports #9441. [nocoverage] Required PR: https://github.com/pulp/pulpcore/pull/1637 fixes #9445 (cherry picked from commit cd9b5785279582297030007f559958829a927084) --- CHANGES/9445.bugfix | 2 ++ pulpcore/app/models/content.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 CHANGES/9445.bugfix diff --git a/CHANGES/9445.bugfix b/CHANGES/9445.bugfix new file mode 100644 index 0000000000..0d6fba9fc6 --- /dev/null +++ b/CHANGES/9445.bugfix @@ -0,0 +1,2 @@ +Taught several more codepaths to order-before-update to avoid deadlocks. +(backported from #9441) diff --git a/pulpcore/app/models/content.py b/pulpcore/app/models/content.py index 64aae6ab0b..7c65026c7d 100644 --- a/pulpcore/app/models/content.py +++ b/pulpcore/app/models/content.py @@ -99,8 +99,11 @@ class BulkTouchQuerySet(models.QuerySet): def touch(self): """ Update the ``timestamp_of_interest`` on all objects of the query. + + We order-by-pk here to avoid deadlocking in high-concurrency + environments. """ - return self.update(timestamp_of_interest=now()) + return self.order_by("pk").update(timestamp_of_interest=now()) class QueryMixin: From 7d0cbdd8e03b9711bb93aabd577d0dc62b577042 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Tue, 28 Sep 2021 13:39:38 -0400 Subject: [PATCH 66/79] Added worker_cleanup to the tasking system All workers that are considered offline will be removed from the database periodically. backports #8931 fixes #9462 (cherry picked from commit 5bbf9596f2d265bac68c764631f26feadc8e992c) --- CHANGES/9462.feature | 2 ++ pulpcore/app/models/task.py | 16 ++++++++++++++++ pulpcore/tasking/pulpcore_worker.py | 28 ++++++++++++++++++++++++---- 3 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 CHANGES/9462.feature diff --git a/CHANGES/9462.feature b/CHANGES/9462.feature new file mode 100644 index 0000000000..153b9ee814 --- /dev/null +++ b/CHANGES/9462.feature @@ -0,0 +1,2 @@ +Added a periodical cleanup to the pulpcore-worker class to keep the `Worker` table clean. +(backported from #8931) diff --git a/pulpcore/app/models/task.py b/pulpcore/app/models/task.py index 27b6efb5c8..0570aa2bbc 100644 --- a/pulpcore/app/models/task.py +++ b/pulpcore/app/models/task.py @@ -89,6 +89,22 @@ def online_workers(self): return self.filter(last_heartbeat__gte=age_threshold, gracefully_stopped=False) + def offline_workers(self): + """ + Returns a queryset of workers meeting the criteria to be considered 'offline' + + To be considered 'offline', a worker must have no recent heartbeat timestamp. + "Recent" is defined here as "within the pulp process timeout interval". + + Returns: + :class:`django.db.models.query.QuerySet`: A query set of the Worker objects which + are considered by Pulp to be 'offline'. + """ + now = timezone.now() + age_threshold = now - timedelta(seconds=settings.WORKER_TTL) + + return self.filter(last_heartbeat__lte=age_threshold) + def missing_workers(self): """ Returns a queryset of workers meeting the criteria to be considered 'missing' diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py index 4a4ef31151..e7df7fa252 100644 --- a/pulpcore/tasking/pulpcore_worker.py +++ b/pulpcore/tasking/pulpcore_worker.py @@ -3,6 +3,7 @@ import json import logging import os +import random import select import signal import socket @@ -27,7 +28,7 @@ ) from django_guid.middleware import GuidMiddleware # noqa: E402: module level not at top of file -from pulpcore.app.models import Task # noqa: E402: module level not at top of file +from pulpcore.app.models import Worker, Task # noqa: E402: module level not at top of file from pulpcore.constants import ( # noqa: E402: module level not at top of file TASK_STATES, @@ -41,6 +42,10 @@ _logger = logging.getLogger(__name__) +random.seed() + +TASK_GRACE_INTERVAL = 3 +WORKER_CLEANUP_INTERVAL = 100 class NewPulpWorker: @@ -50,6 +55,10 @@ def __init__(self): self.heartbeat_period = settings.WORKER_TTL / 3 self.cursor = connection.cursor() self.worker = handle_worker_heartbeat(self.name) + self.task_grace_timeout = 0 + self.worker_cleanup_countdown = random.randint( + WORKER_CLEANUP_INTERVAL / 10, WORKER_CLEANUP_INTERVAL + ) # Add a file descriptor to trigger select on signals self.sentinel, sentinel_w = os.pipe() @@ -70,11 +79,22 @@ def shutdown(self): self.worker.delete() _logger.info(f"Worker {self.name} was shut down.") + def worker_cleanup(self): + qs = Worker.objects.offline_workers() + if qs: + for worker in qs: + _logger.info(f"Clean offline worker {worker.name}.") + worker.delete() + def beat(self): if self.worker.last_heartbeat < timezone.now() - timedelta(seconds=self.heartbeat_period): self.worker = handle_worker_heartbeat(self.name) if self.shutdown_requested: - self.task_grace -= 1 + self.task_grace_timeout -= 1 + self.worker_cleanup_countdown -= 1 + if self.worker_cleanup_countdown <= 0: + self.worker_cleanup_countdown = WORKER_CLEANUP_INTERVAL + self.worker_cleanup() def notify_workers(self): self.cursor.execute("NOTIFY pulp_worker_wakeup") @@ -166,7 +186,7 @@ def supervise_task(self, task): This function must only be called while holding the lock for that task.""" - self.task_grace = 3 + self.task_grace_timeout = TASK_GRACE_INTERVAL self.cursor.execute("LISTEN pulp_worker_cancel") task.worker = self.worker task.save(update_fields=["worker"]) @@ -199,7 +219,7 @@ def supervise_task(self, task): if self.sentinel in r: os.read(self.sentinel, 256) if self.shutdown_requested: - if self.task_grace > 0: + if self.task_grace_timeout > 0: _logger.info( f"Worker shutdown requested, waiting for task {task.pk} to finish." ) From 7d796b24a74fcdce2bccfcc66f3366e7570453e1 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Fri, 23 Jul 2021 18:33:18 +0200 Subject: [PATCH 67/79] Fix gettext for logging from pulpcore-worker [noissue] --- pulpcore/tasking/pulpcore_worker.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py index e7df7fa252..63e515cd4a 100644 --- a/pulpcore/tasking/pulpcore_worker.py +++ b/pulpcore/tasking/pulpcore_worker.py @@ -1,3 +1,5 @@ +from gettext import gettext as _ + import asyncio import importlib import json @@ -72,18 +74,19 @@ def _signal_handler(self, thesignal, frame): signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) - _logger.info(f"Worker {self.name} was requested to shut down.") + _logger.info(_("Worker %s was requested to shut down."), self.name) + self.shutdown_requested = True def shutdown(self): self.worker.delete() - _logger.info(f"Worker {self.name} was shut down.") + _logger.info(_("Worker %s was shut down."), self.name) def worker_cleanup(self): qs = Worker.objects.offline_workers() if qs: for worker in qs: - _logger.info(f"Clean offline worker {worker.name}.") + _logger.info(_("Clean offline worker %s."), worker.name) worker.delete() def beat(self): @@ -107,7 +110,7 @@ def cancel_abandoned_task(self, task): Return ``True`` if the task was actually canceled, ``False`` otherwise. """ # A task is considered abandoned when in running state, but no worker holds its lock - _logger.info(f"Cleaning up and canceling Task {task.pk}") + _logger.info(_("Cleaning up and canceling Task %s"), task.pk) Task.objects.filter(pk=task.pk, state=TASK_STATES.RUNNING).update( state=TASK_STATES.CANCELING ) @@ -159,7 +162,7 @@ def iter_tasks(self): def sleep(self): """Wait for signals on the wakeup channel while heart beating.""" - _logger.debug(f"Worker {self.name} entering sleep state.") + _logger.debug(_("Worker %s entering sleep state."), self.name) # Subscribe to "pulp_worker_wakeup" self.cursor.execute("LISTEN pulp_worker_wakeup") while not self.shutdown_requested: @@ -210,7 +213,7 @@ def supervise_task(self, task): ) ): connection.connection.notifies.clear() - _logger.info(f"Received signal to cancel current task {task.pk}.") + _logger.info(_("Received signal to cancel current task."), task.pk) os.kill(task_process.pid, signal.SIGUSR1) break if task_process.sentinel in r: @@ -221,10 +224,10 @@ def supervise_task(self, task): if self.shutdown_requested: if self.task_grace_timeout > 0: _logger.info( - f"Worker shutdown requested, waiting for task {task.pk} to finish." + _("Worker shutdown requested, waiting for task %s to finish."), task.pk ) else: - _logger.info(f"Aborting current task {task.pk} due to worker shutdown.") + _logger.info(_("Aborting current task %s due to worker shutdown."), task.pk) os.kill(task_process.pid, signal.SIGUSR1) break task_process.join() @@ -280,7 +283,7 @@ def _perform_task(task_pk, task_working_dir_rel_path): _set_current_user(user) GuidMiddleware.set_guid(task.logging_cid) try: - _logger.info("Starting task {}".format(task.pk)) + _logger.info(_("Starting task %s"), task.pk) # Execute task module_name, function_name = task.name.rsplit(".", 1) @@ -291,16 +294,16 @@ def _perform_task(task_pk, task_working_dir_rel_path): os.chdir(task_working_dir_rel_path) result = func(*args, **kwargs) if asyncio.iscoroutine(result): - _logger.debug("Task is coroutine {}".format(task.pk)) + _logger.debug(_("Task is coroutine %s"), task.pk) loop = asyncio.get_event_loop() loop.run_until_complete(result) except Exception: exc_type, exc, tb = sys.exc_info() task.set_failed(exc, tb) - _logger.info("Task {} failed ({})".format(task.pk, exc)) + _logger.info(_("Task %s failed (%s)"), task.pk, exc) _logger.info("\n".join(traceback.format_list(traceback.extract_tb(tb)))) else: task.set_completed() - _logger.info("Task completed {}".format(task.pk)) + _logger.info(_("Task completed %s"), task.pk) os.environ.pop("PULP_TASK_ID") From b89f5795955f70f523969a3f0713d8d47854e6a5 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Tue, 28 Sep 2021 13:50:48 -0400 Subject: [PATCH 68/79] Mark abandoned tasks as "failed" backports: #9427 fixes #9453 (cherry picked from commit 6fb91091aa84e56f7b1711a59099f8312b6efb56) --- CHANGES/9453.bugfix | 2 ++ pulpcore/tasking/pulpcore_worker.py | 36 +++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 CHANGES/9453.bugfix diff --git a/CHANGES/9453.bugfix b/CHANGES/9453.bugfix new file mode 100644 index 0000000000..afec485610 --- /dev/null +++ b/CHANGES/9453.bugfix @@ -0,0 +1,2 @@ +Changed the pulpcore-worker to mark abandoned tasks as "failed" instead of "canceled". +(backported from #9247) diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py index 63e515cd4a..0d4174fee1 100644 --- a/pulpcore/tasking/pulpcore_worker.py +++ b/pulpcore/tasking/pulpcore_worker.py @@ -102,26 +102,39 @@ def beat(self): def notify_workers(self): self.cursor.execute("NOTIFY pulp_worker_wakeup") - def cancel_abandoned_task(self, task): + def cancel_abandoned_task(self, task, final_state, reason=None): """Cancel and clean up an abandoned task. - This function must only be called while holding the lock for that task. + This function must only be called while holding the lock for that task. It is a no-op if + the task is neither in "running" nor "canceling" state. Return ``True`` if the task was actually canceled, ``False`` otherwise. """ # A task is considered abandoned when in running state, but no worker holds its lock - _logger.info(_("Cleaning up and canceling Task %s"), task.pk) Task.objects.filter(pk=task.pk, state=TASK_STATES.RUNNING).update( state=TASK_STATES.CANCELING ) task.refresh_from_db() if task.state == TASK_STATES.CANCELING: + if reason: + _logger.info( + _("Cleaning up task %s and marking as %s. Reason: %s"), + task.pk, + final_state, + reason, + ) + else: + _logger.info(_("Cleaning up task %s and marking as %s."), task.pk, final_state) _delete_incomplete_resources(task) if task.reserved_resources_record: self.notify_workers() - Task.objects.filter(pk=task.pk, state=TASK_STATES.CANCELING).update( - state=TASK_STATES.CANCELED - ) + task_data = { + "state": final_state, + "finished_at": timezone.now(), + } + if reason: + task_data["error"] = {"reason": reason} + Task.objects.filter(pk=task.pk, state=TASK_STATES.CANCELING).update(**task_data) return True return False @@ -142,7 +155,9 @@ def iter_tasks(self): task.refresh_from_db() if task.state in [TASK_STATES.RUNNING, TASK_STATES.CANCELING]: # A running task without a lock must be abandoned - if self.cancel_abandoned_task(task): + if self.cancel_abandoned_task( + task, TASK_STATES.FAILED, "Worker has gone missing." + ): # Continue looking for the next task # without considering this tasks resources # as we just released them @@ -193,6 +208,8 @@ def supervise_task(self, task): self.cursor.execute("LISTEN pulp_worker_cancel") task.worker = self.worker task.save(update_fields=["worker"]) + cancel_state = None + cancel_reason = None with TemporaryDirectory(dir=".") as task_working_dir_rel_path: task_process = Process(target=_perform_task, args=(task.pk, task_working_dir_rel_path)) task_process.start() @@ -215,6 +232,7 @@ def supervise_task(self, task): connection.connection.notifies.clear() _logger.info(_("Received signal to cancel current task."), task.pk) os.kill(task_process.pid, signal.SIGUSR1) + cancel_state = TASK_STATES.CANCELED break if task_process.sentinel in r: if not task_process.is_alive(): @@ -229,8 +247,12 @@ def supervise_task(self, task): else: _logger.info(_("Aborting current task %s due to worker shutdown."), task.pk) os.kill(task_process.pid, signal.SIGUSR1) + cancel_state = TASK_STATES.FAILED + cancel_reason = "Aborted during worker shutdown." break task_process.join() + if cancel_state: + self.cancel_abandoned_task(task, cancel_state, cancel_reason) if task.reserved_resources_record: self.notify_workers() self.cursor.execute("UNLISTEN pulp_worker_cancel") From 9bcae2e6377f3f3edf90edfc44eebc366f4d14c5 Mon Sep 17 00:00:00 2001 From: Grant Gainey Date: Thu, 23 Sep 2021 13:44:53 -0400 Subject: [PATCH 69/79] Fix occasional deadlocks when doing multiple similar syncs concurrently. Forcing deadlocks requires a lot of time and pulpcore-workers running. There is therefore no specific CI test for this, but there is a reproducer script that will force deadlocks to happen (and show that they're fixed) here: https://github.com/ggainey/pulp_startup/blob/main/8750_deadlocks/file_repro.sh backports #8750 fixes #9379 (cherry picked from commit 6af3519b4931bd34c49653a80c52fd1c7cd08821) --- CHANGES/9379.bugfix | 3 +++ pulpcore/app/models/content.py | 30 +++++++++++++++++++++++ pulpcore/plugin/stages/artifact_stages.py | 9 +++++-- pulpcore/plugin/stages/content_stages.py | 9 +++++++ 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 CHANGES/9379.bugfix diff --git a/CHANGES/9379.bugfix b/CHANGES/9379.bugfix new file mode 100644 index 0000000000..b79134cf6f --- /dev/null +++ b/CHANGES/9379.bugfix @@ -0,0 +1,3 @@ +Ordered several ContentStages paths to fix deadlocks in high-concurrency scenarios. + +(backported from #8750) diff --git a/pulpcore/app/models/content.py b/pulpcore/app/models/content.py index 7c65026c7d..6cc4234002 100644 --- a/pulpcore/app/models/content.py +++ b/pulpcore/app/models/content.py @@ -77,6 +77,7 @@ def bulk_get_or_create(self, objs, batch_size=None): Returns: List of instances that were inserted into the database. """ + objs = list(objs) try: with transaction.atomic(): @@ -554,6 +555,35 @@ class ContentArtifact(BaseModel, QueryMixin): class Meta: unique_together = ("content", "relative_path") + @staticmethod + def sort_key(ca): + """ + Static method for defining a sort-key for a specified ContentArtifact. + + Sorting lists of ContentArtifacts is critical for avoiding deadlocks in high-concurrency + environments, when multiple workers may be operating on similar sets of content at the + same time. Providing a stable sort-order becomes problematic when the CAs in question + haven't been persisted - in that case, pulp_id can't be relied on, as it will change + when the object is stored in the DB and its "real" key is generated. + + This method produces a key based on the content/artifact represented by the CA. + + Args: + ca (:class:`~pulpcore.plugin.models.ContentArtifact`): The CA we need a key for + + Returns: + a tuple of (str(content-key), str(artifact-key)) that can be reliably sorted on + """ + c_key = "" + a_key = "" + # It's possible to only have one of content/artifact - handle that + if ca.content: + # Some key-fields aren't str, handle that + c_key = "".join(map(str, ca.content.natural_key())) + if ca.artifact: + a_key = str(ca.artifact.sha256) + return c_key, a_key + class RemoteArtifact(BaseModel, QueryMixin): """ diff --git a/pulpcore/plugin/stages/artifact_stages.py b/pulpcore/plugin/stages/artifact_stages.py index 950045c70b..b0eeaf80fe 100644 --- a/pulpcore/plugin/stages/artifact_stages.py +++ b/pulpcore/plugin/stages/artifact_stages.py @@ -384,10 +384,15 @@ def _handle_remote_artifacts(self, batch): key = f"{content_artifact.pk}-{d_artifact.remote.pk}" ras_to_create[key] = remote_artifact + # Make sure we create/update RemoteArtifacts in a stable order, to help + # prevent deadlocks in high-concurrency environments. We can rely on the + # Artifact sha256 for our ordering. if ras_to_create: - RemoteArtifact.objects.bulk_create(list(ras_to_create.values())) + ras_to_create_ordered = sorted(list(ras_to_create.values()), key=lambda x: x.sha256) + RemoteArtifact.objects.bulk_create(ras_to_create_ordered) if ras_to_update: - RemoteArtifact.objects.bulk_update(list(ras_to_update.values()), fields=["url"]) + ras_to_update_ordered = sorted(list(ras_to_update.values()), key=lambda x: x.sha256) + RemoteArtifact.objects.bulk_update(ras_to_update_ordered, fields=["url"]) @staticmethod def _create_remote_artifact(d_artifact, content_artifact): diff --git a/pulpcore/plugin/stages/content_stages.py b/pulpcore/plugin/stages/content_stages.py index f919d199d2..f0e6b580bb 100644 --- a/pulpcore/plugin/stages/content_stages.py +++ b/pulpcore/plugin/stages/content_stages.py @@ -104,6 +104,10 @@ async def run(self): with transaction.atomic(): await self._pre_save(batch) + # Process the batch in dc.content.natural_keys order. + # This prevents deadlocks when we're processing the same/similar content + # in concurrent workers. + batch.sort(key=lambda x: "".join(map(str, x.content.natural_key()))) for d_content in batch: # Are we saving to the database for the first time? content_already_saved = not d_content.content._state.adding @@ -148,6 +152,11 @@ async def run(self): # Maybe remove dict elements after to reduce memory? content_artifact.artifact = to_update_ca_artifact[key] to_update_ca_bulk.append(content_artifact) + # Sort the lists we're about to do bulk updates/creates on. + # We know to_update_ca_bulk entries already are in the DB, so we can enforce + # order just using pulp_id. + to_update_ca_bulk.sort(key=lambda x: x.pulp_id) + content_artifact_bulk.sort(key=lambda x: ContentArtifact.sort_key(x)) ContentArtifact.objects.bulk_update(to_update_ca_bulk, ["artifact"]) ContentArtifact.objects.bulk_get_or_create(content_artifact_bulk) await self._post_save(batch) From 3958faae1dc6d6dc57c13cb84867f0964d65875d Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 29 Sep 2021 18:12:29 +0000 Subject: [PATCH 70/79] 3.14.7 changelog [noissue] --- CHANGES.rst | 43 +++++++++++++++++++++++++++++++++++++++++++ CHANGES/9379.bugfix | 3 --- CHANGES/9400.bugfix | 2 -- CHANGES/9401.bugfix | 2 -- CHANGES/9440.bugfix | 2 -- CHANGES/9445.bugfix | 2 -- CHANGES/9453.bugfix | 2 -- CHANGES/9462.feature | 2 -- 8 files changed, 43 insertions(+), 15 deletions(-) delete mode 100644 CHANGES/9379.bugfix delete mode 100644 CHANGES/9400.bugfix delete mode 100644 CHANGES/9401.bugfix delete mode 100644 CHANGES/9440.bugfix delete mode 100644 CHANGES/9445.bugfix delete mode 100644 CHANGES/9453.bugfix delete mode 100644 CHANGES/9462.feature diff --git a/CHANGES.rst b/CHANGES.rst index 946b4c8a62..4d8cf79ee9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,49 @@ Changelog .. towncrier release notes start +3.14.7 (2021-09-29) +=================== +REST API +-------- + +Features +~~~~~~~~ + +- Added a periodical cleanup to the pulpcore-worker class to keep the `Worker` table clean. + (backported from #8931) + `#9462 `_ + + +Bugfixes +~~~~~~~~ + +- Ordered several ContentStages paths to fix deadlocks in high-concurrency scenarios. + + (backported from #8750) + `#9379 `_ +- Fixed an issue where on_demand content might not be downloaded properly if the remote URL was changed (even if re-synced). + (backported from #9395) + `#9400 `_ +- Fixed the repository modify endpoint performance problems. + (backported from #9266) + `#9401 `_ +- Taught a remote-artifact error path to not assume 'filename' was valid for all content. + (backported from #9427) + `#9440 `_ +- Taught several more codepaths to order-before-update to avoid deadlocks. + (backported from #9441) + `#9445 `_ +- Changed the pulpcore-worker to mark abandoned tasks as "failed" instead of "canceled". + (backported from #9247) + `#9453 `_ + + +Plugin API +---------- + +No significant changes. + + 3.14.6 (2021-09-02) =================== REST API diff --git a/CHANGES/9379.bugfix b/CHANGES/9379.bugfix deleted file mode 100644 index b79134cf6f..0000000000 --- a/CHANGES/9379.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Ordered several ContentStages paths to fix deadlocks in high-concurrency scenarios. - -(backported from #8750) diff --git a/CHANGES/9400.bugfix b/CHANGES/9400.bugfix deleted file mode 100644 index 0fa363ee39..0000000000 --- a/CHANGES/9400.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed an issue where on_demand content might not be downloaded properly if the remote URL was changed (even if re-synced). -(backported from #9395) diff --git a/CHANGES/9401.bugfix b/CHANGES/9401.bugfix deleted file mode 100644 index ffe27bd128..0000000000 --- a/CHANGES/9401.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fixed the repository modify endpoint performance problems. -(backported from #9266) diff --git a/CHANGES/9440.bugfix b/CHANGES/9440.bugfix deleted file mode 100644 index 8a386c683b..0000000000 --- a/CHANGES/9440.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Taught a remote-artifact error path to not assume 'filename' was valid for all content. -(backported from #9427) diff --git a/CHANGES/9445.bugfix b/CHANGES/9445.bugfix deleted file mode 100644 index 0d6fba9fc6..0000000000 --- a/CHANGES/9445.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Taught several more codepaths to order-before-update to avoid deadlocks. -(backported from #9441) diff --git a/CHANGES/9453.bugfix b/CHANGES/9453.bugfix deleted file mode 100644 index afec485610..0000000000 --- a/CHANGES/9453.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Changed the pulpcore-worker to mark abandoned tasks as "failed" instead of "canceled". -(backported from #9247) diff --git a/CHANGES/9462.feature b/CHANGES/9462.feature deleted file mode 100644 index 153b9ee814..0000000000 --- a/CHANGES/9462.feature +++ /dev/null @@ -1,2 +0,0 @@ -Added a periodical cleanup to the pulpcore-worker class to keep the `Worker` table clean. -(backported from #8931) From 173a9f0bf308a0ef7cb6835e543a09bbfe2c4722 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 29 Sep 2021 18:12:30 +0000 Subject: [PATCH 71/79] Release 3.14.7 Redmine Query: https://pulp.plan.io/issues?set_filter=1&status_id=*&issue_id=9462,9401,9440,9379,9400,9453,9445 Redmine Milestone: https://pulp.plan.io/versions/302.json [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 920ec51e56..3ac325c1f3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.7.dev +current_version = 3.14.7 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 56a2a0f0bd..760f0b0f0d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.7.dev" +version = "3.14.7" # The full version, including alpha/beta/rc tags. -release = "3.14.7.dev" +release = "3.14.7" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 77976dee82..a641c324a7 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.7.dev" + version = "3.14.7" def ready(self): super().ready() diff --git a/setup.py b/setup.py index 82a892dc8f..481f4e47f1 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.7.dev", + version="3.14.7", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 1c789620defd0ec6c852be5ea131437e1f3c567e Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 29 Sep 2021 18:12:30 +0000 Subject: [PATCH 72/79] Bump to 3.14.8.dev [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3ac325c1f3..e8f28c7c16 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.7 +current_version = 3.14.8.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 760f0b0f0d..dc862bca68 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.7" +version = "3.14.8.dev" # The full version, including alpha/beta/rc tags. -release = "3.14.7" +release = "3.14.8.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index a641c324a7..03f7d856cc 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.7" + version = "3.14.8.dev" def ready(self): super().ready() diff --git a/setup.py b/setup.py index 481f4e47f1..45a50036ca 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.7", + version="3.14.8.dev", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 196d3334f6edc67e57b9d43479e391e20004c57e Mon Sep 17 00:00:00 2001 From: Ina Panova Date: Tue, 5 Oct 2021 18:49:06 +0200 Subject: [PATCH 73/79] Improved messaging around timeout requests. closes #9491 backports #9301 (cherry picked from commit 8a1db3b9c0bd44f6ea76eeb2a2cdbcd673b199c3) --- CHANGES/9491.bugfix | 1 + pulpcore/download/base.py | 6 +++++- pulpcore/download/http.py | 8 +++++++- pulpcore/exceptions/__init__.py | 3 ++- pulpcore/exceptions/base.py | 19 +++++++++++++++++++ 5 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 CHANGES/9491.bugfix diff --git a/CHANGES/9491.bugfix b/CHANGES/9491.bugfix new file mode 100644 index 0000000000..a8adbb03e2 --- /dev/null +++ b/CHANGES/9491.bugfix @@ -0,0 +1 @@ +Improved messaging around timeout requests. (Backported from https://pulp.plan.io/issues/9301). diff --git a/pulpcore/download/base.py b/pulpcore/download/base.py index 7665aac62d..669f6cc798 100644 --- a/pulpcore/download/base.py +++ b/pulpcore/download/base.py @@ -11,6 +11,7 @@ from pulpcore.exceptions import ( DigestValidationError, SizeValidationError, + TimeoutException, UnsupportedDigestValidationError, ) @@ -240,7 +241,10 @@ async def run(self, extra_data=None): """ async with self.semaphore: - return await self._run(extra_data=extra_data) + try: + return await self._run(extra_data=extra_data) + except asyncio.TimeoutError: + raise TimeoutException(self.url) async def _run(self, extra_data=None): """ diff --git a/pulpcore/download/http.py b/pulpcore/download/http.py index 9017550536..f4e51274f6 100644 --- a/pulpcore/download/http.py +++ b/pulpcore/download/http.py @@ -1,9 +1,11 @@ import logging import aiohttp +import asyncio import backoff from .base import BaseDownloader, DownloadResult +from pulpcore.exceptions import TimeoutException log = logging.getLogger(__name__) @@ -242,6 +244,7 @@ async def run(self, extra_data=None): aiohttp.ClientResponseError, aiohttp.ServerDisconnectedError, TimeoutError, + TimeoutException, ) async with self.semaphore: @@ -253,7 +256,10 @@ async def run(self, extra_data=None): giveup=http_giveup_handler, ) async def download_wrapper(): - return await self._run(extra_data=extra_data) + try: + return await self._run(extra_data=extra_data) + except asyncio.TimeoutError: + raise TimeoutException(self.url) return await download_wrapper() diff --git a/pulpcore/exceptions/__init__.py b/pulpcore/exceptions/__init__.py index 03d452cfd9..9ce472d159 100644 --- a/pulpcore/exceptions/__init__.py +++ b/pulpcore/exceptions/__init__.py @@ -1,7 +1,8 @@ from .base import ( # noqa + AdvisoryLockError, PulpException, ResourceImmutableError, - AdvisoryLockError, + TimeoutException, exception_to_dict, ) from .http import MissingResource # noqa diff --git a/pulpcore/exceptions/base.py b/pulpcore/exceptions/base.py index f7aca9eefc..0cb7cdcfce 100644 --- a/pulpcore/exceptions/base.py +++ b/pulpcore/exceptions/base.py @@ -66,3 +66,22 @@ def __str__(self): class AdvisoryLockError(Exception): """Exception to signal that a lock could not be acquired.""" + + +class TimeoutException(PulpException): + """ + Exception to signal timeout error. + """ + + def __init__(self, url): + """ + :param url: the url the download for timed out + :type url: str + """ + super().__init__("PLP0005") + self.url = url + + def __str__(self): + return _( + "Request timed out for {}. Increasing the total_timeout value on the remote might help." + ).format(self.url) From afa329c8ddd8358b304209f1ff0e48965c65adb4 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 6 Oct 2021 13:04:36 +0000 Subject: [PATCH 74/79] 3.14.8 changelog [noissue] --- CHANGES.rst | 18 ++++++++++++++++++ CHANGES/9491.bugfix | 1 - 2 files changed, 18 insertions(+), 1 deletion(-) delete mode 100644 CHANGES/9491.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index 4d8cf79ee9..3277eae96c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,24 @@ Changelog .. towncrier release notes start +3.14.8 (2021-10-06) +=================== +REST API +-------- + +Bugfixes +~~~~~~~~ + +- Improved messaging around timeout requests. (Backported from https://pulp.plan.io/issues/9301). + `#9491 `_ + + +Plugin API +---------- + +No significant changes. + + 3.14.7 (2021-09-29) =================== REST API diff --git a/CHANGES/9491.bugfix b/CHANGES/9491.bugfix deleted file mode 100644 index a8adbb03e2..0000000000 --- a/CHANGES/9491.bugfix +++ /dev/null @@ -1 +0,0 @@ -Improved messaging around timeout requests. (Backported from https://pulp.plan.io/issues/9301). From 2cef46fdc3ea6cef712d00b066aed15eea1f2135 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 6 Oct 2021 13:04:37 +0000 Subject: [PATCH 75/79] Release 3.14.8 Redmine Query: https://pulp.plan.io/issues?set_filter=1&status_id=*&issue_id=9491 Redmine Milestone: https://pulp.plan.io/versions/313.json [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e8f28c7c16..bcb0c634f3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.8.dev +current_version = 3.14.8 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index dc862bca68..3d3a43c4a3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.8.dev" +version = "3.14.8" # The full version, including alpha/beta/rc tags. -release = "3.14.8.dev" +release = "3.14.8" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 03f7d856cc..05c8f2535b 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.8.dev" + version = "3.14.8" def ready(self): super().ready() diff --git a/setup.py b/setup.py index 45a50036ca..31e410cad9 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.8.dev", + version="3.14.8", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 14de38a6c4fa4e06c3c21dc547d0061172997369 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Wed, 6 Oct 2021 13:04:37 +0000 Subject: [PATCH 76/79] Bump to 3.14.9.dev [noissue] --- .bumpversion.cfg | 2 +- docs/conf.py | 4 ++-- pulpcore/app/apps.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index bcb0c634f3..e178a0e7d3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.14.8 +current_version = 3.14.9.dev commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? diff --git a/docs/conf.py b/docs/conf.py index 3d3a43c4a3..0f026a2097 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = "3.14.8" +version = "3.14.9.dev" # The full version, including alpha/beta/rc tags. -release = "3.14.8" +release = "3.14.9.dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index 05c8f2535b..a86949d814 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -185,7 +185,7 @@ class PulpAppConfig(PulpPluginAppConfig): label = "core" # The version of this app - version = "3.14.8" + version = "3.14.9.dev" def ready(self): super().ready() diff --git a/setup.py b/setup.py index 31e410cad9..5905eafc5d 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="pulpcore", - version="3.14.8", + version="3.14.9.dev", description="Pulp Django Application and Related Modules", long_description=long_description, long_description_content_type="text/markdown", From 90aa8224e8f9b783048bcb46190bafef139eba11 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Mon, 25 Oct 2021 13:07:46 +0200 Subject: [PATCH 77/79] Fix unittests SimpleTestCase does not allow database access. [noissue] --- pulpcore/tests/unit/models/test_content.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulpcore/tests/unit/models/test_content.py b/pulpcore/tests/unit/models/test_content.py index 434749fa69..137a0eba89 100644 --- a/pulpcore/tests/unit/models/test_content.py +++ b/pulpcore/tests/unit/models/test_content.py @@ -6,7 +6,7 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.conf import settings -from django.test import SimpleTestCase, TestCase +from django.test import TestCase from pulpcore.plugin.exceptions import ( UnsupportedDigestValidationError, MissingDigestValidationError, @@ -82,7 +82,7 @@ def test_read_temp_file(self): assert b"temp file test" in temp_file.file.read() -class ArtifactAlgorithmTestCase(SimpleTestCase): +class ArtifactAlgorithmTestCase(TestCase): @mock.patch( "pulpcore.app.models.Artifact.FORBIDDEN_DIGESTS", new_callable=mock.PropertyMock, From 1ebf39b0cfb29eb28a7608b5a3670656ff98beb1 Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Wed, 27 Oct 2021 15:28:20 -0400 Subject: [PATCH 78/79] Update CI configs [noissue] --- .github/template_gitref | 2 +- .github/workflows/ci.yml | 14 ++++++-------- .github/workflows/create-branch.yml | 2 +- .github/workflows/nightly.yml | 14 ++++++-------- .github/workflows/release.yml | 7 +++---- .github/workflows/scripts/before_script.sh | 2 +- .github/workflows/scripts/install.sh | 4 ++-- .github/workflows/scripts/script.sh | 21 +++++++++++---------- 8 files changed, 31 insertions(+), 35 deletions(-) diff --git a/.github/template_gitref b/.github/template_gitref index 9743715921..f8622a685b 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.04.08-122-gefc11dd +2021.08.26-32-g7f45118 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94ef2212bd..2b9219c9e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,8 +109,7 @@ jobs: - name: Install httpie run: | echo ::group::HTTPIE - sudo apt-get update -yq - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie + pip install httpie echo ::endgroup:: echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV @@ -190,8 +189,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - name: After failure - if: failure() + - name: Logs + if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true @@ -229,8 +228,7 @@ jobs: - name: Install httpie run: | echo ::group::HTTPIE - sudo apt-get update -yq - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie + pip install httpie echo ::endgroup:: echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV @@ -311,8 +309,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - name: After failure - if: failure() + - name: Logs + if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 8eb970ed9c..27a5258632 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -49,7 +49,7 @@ jobs: - name: Verify that branch name matches current version string on master branch run: | - X_Y_VERSION=$(grep version setup.py | sed -rn 's/version="(.*)\.0\.dev",/\1/p' | awk '{$1=$1};1') + X_Y_VERSION=$(grep version setup.py | sed -rn 's/version="(.*)\.0((a[0-9]+)|(b[0-9]+))?\.dev",/\1/p' | awk '{$1=$1};1') if [[ "$X_Y_VERSION" != "${{ github.event.inputs.name }}" ]] then echo "Branch name doesn't match the current version string $X_Y_VERSION." diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 824285d79b..51e10c7d9a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -40,8 +40,7 @@ jobs: - name: Install httpie run: | echo ::group::HTTPIE - sudo apt-get update -yq - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie + pip install httpie echo ::endgroup:: echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV @@ -147,8 +146,8 @@ jobs: name: docs.tar path: docs/docs.tar - - name: After failure - if: failure() + - name: Logs + if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true @@ -185,8 +184,7 @@ jobs: - name: Install httpie run: | echo ::group::HTTPIE - sudo apt-get update -yq - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie + pip install httpie echo ::endgroup:: echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV @@ -296,8 +294,8 @@ jobs: .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} - - name: After failure - if: failure() + - name: Logs + if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0bcde5ac27..ec97eeb931 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -112,8 +112,7 @@ jobs: - name: Install httpie run: | echo ::group::HTTPIE - sudo apt-get update -yq - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie + pip install httpie echo ::endgroup:: echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV @@ -219,8 +218,8 @@ jobs: name: docs.tar path: docs/docs.tar - - name: After failure - if: failure() + - name: Logs + if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 3cce04b81e..126ae72301 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -29,7 +29,7 @@ tail -v -n +1 .ci/ansible/vars/main.yaml echo "PULP CONFIG:" tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json -if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == 'azure' || "$TEST" == "plugin-from-pypi" ]]; then +if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == 'azure' || "$TEST" == "plugin-from-pypi" || "$TEST" == "generate-bindings" ]]; then # Many functional tests require these cmd_prefix dnf install -yq lsof which dnf-plugins-core fi diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 07e8cba9b7..684881f139 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -68,7 +68,7 @@ pulp_container_tag: python36 VARSYAML -if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "upgrade" || "$TEST" == "azure" || "$TEST" == "s3" || "$TEST" == "plugin-from-pypi" ]]; then +if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "upgrade" || "$TEST" == "azure" || "$TEST" == "s3" || "$TEST" == "plugin-from-pypi" || "$TEST" == "generate-bindings" ]]; then sed -i -e '/^services:/a \ - name: pulp-fixtures\ image: docker.io/pulp/pulp-fixtures:latest\ @@ -94,7 +94,7 @@ ansible-playbook build_container.yaml ansible-playbook start_container.yaml if [ "$TEST" = "azure" ]; then - AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://pulp-azurite:10000/devstoreaccount1;' + AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING fi diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 752b9e013b..d06a124256 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -84,16 +84,13 @@ fi cd $REPO_ROOT if [[ "$TEST" = 'bindings' ]]; then - python $REPO_ROOT/.ci/assets/bindings/test_bindings.py -fi - -if [[ "$TEST" = 'bindings' ]]; then - if [ ! -f $REPO_ROOT/.ci/assets/bindings/test_bindings.rb ]; then - exit - else + if [ -f $REPO_ROOT/.ci/assets/bindings/test_bindings.py ]; then + python $REPO_ROOT/.ci/assets/bindings/test_bindings.py + fi + if [ -f $REPO_ROOT/.ci/assets/bindings/test_bindings.rb ]; then ruby $REPO_ROOT/.ci/assets/bindings/test_bindings.rb - exit fi + exit fi cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" @@ -103,10 +100,14 @@ cmd_prefix pip3 install -r /tmp/unittest_requirements.txt echo "Checking for uncommitted migrations..." cmd_prefix bash -c "django-admin makemigrations --check --dry-run" -# Run unit tests. -cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.6/site-packages/pulpcore/tests/unit/" +if [[ "$TEST" != "upgrade" ]]; then + # Run unit tests. + cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.6/site-packages/pulpcore/tests/unit/" +fi # Run functional tests +export PYTHONPATH=$REPO_ROOT/../pulp_file${PYTHONPATH:+:${PYTHONPATH}} +export PYTHONPATH=$REPO_ROOT/../pulp-certguard${PYTHONPATH:+:${PYTHONPATH}} export PYTHONPATH=$REPO_ROOT${PYTHONPATH:+:${PYTHONPATH}} From 72d9bab98728878573d6918dda10235d598a551a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Mar 2023 11:58:52 +0000 Subject: [PATCH 79/79] [noissue]: Update drf-access-policy requirement from ~=0.9.0 to ~=1.5.0 Updates the requirements on [drf-access-policy](https://github.com/rsinger86/drf-access-policy) to permit the latest version. - [Release notes](https://github.com/rsinger86/drf-access-policy/releases) - [Commits](https://github.com/rsinger86/drf-access-policy/compare/0.9.0...1.5.0) --- updated-dependencies: - dependency-name: drf-access-policy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f3d5784ddd..998bb6f216 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ django-import-export~=2.5.0 django-lifecycle~=0.9.1 djangorestframework~=3.12.4 djangorestframework-queryfields~=1.0.0 -drf-access-policy~=0.9.0 +drf-access-policy~=1.5.0 drf-nested-routers==0.93.3 drf-spectacular==0.17.3 dynaconf~=3.1.5