diff --git a/CHANGES/4776.bugfix b/CHANGES/4776.bugfix new file mode 100644 index 00000000000..2be7924eefb --- /dev/null +++ b/CHANGES/4776.bugfix @@ -0,0 +1,2 @@ +Fixed a bug in the repository version repair API which would present itself when using Pulp with S3 +and Azure storage backends. diff --git a/CHANGES/4806.bugfix b/CHANGES/4806.bugfix new file mode 100644 index 00000000000..2be7924eefb --- /dev/null +++ b/CHANGES/4806.bugfix @@ -0,0 +1,2 @@ +Fixed a bug in the repository version repair API which would present itself when using Pulp with S3 +and Azure storage backends. diff --git a/pulpcore/app/tasks/repository.py b/pulpcore/app/tasks/repository.py index f949d07d290..17d441e166a 100644 --- a/pulpcore/app/tasks/repository.py +++ b/pulpcore/app/tasks/repository.py @@ -1,4 +1,5 @@ from concurrent.futures import ThreadPoolExecutor +from contextvars import copy_context from gettext import gettext as _ from logging import getLogger import asyncio @@ -96,10 +97,12 @@ async def _repair_ca(content_artifact, repaired=None): def _verify_artifact(artifact): + domain = get_domain() + storage = domain.get_storage() + assert domain.pk == artifact.pulp_domain_id try: - # verify files digest hasher = hashlib.sha256() - with artifact.file as fp: + with storage.open(artifact.file.name) as fp: for chunk in fp.chunks(CHUNK_SIZE): hasher.update(chunk) return hasher.hexdigest() == artifact.sha256 @@ -144,7 +147,7 @@ async def _repair_artifacts_for_content(subset=None, verify_checksums=True): # Should stay in (an) executor so that at least it doesn't completely block # downloads. valid = await loop.run_in_executor( - checksum_executor, _verify_artifact, artifact + checksum_executor, copy_context().run, _verify_artifact, artifact ) if not valid: await corrupted.aincrement() diff --git a/pulpcore/tests/functional/api/using_plugin/test_repair.py b/pulpcore/tests/functional/api/using_plugin/test_repair.py index bc6a43ae298..c100507a953 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_repair.py +++ b/pulpcore/tests/functional/api/using_plugin/test_repair.py @@ -1,29 +1,14 @@ import pytest -import os +from django.core.files.storage import default_storage from random import sample from pulpcore.client.pulpcore import Repair from pulpcore.client.pulp_file import RepositorySyncURL -from pulpcore.app import settings - from pulpcore.tests.functional.utils import get_files_in_manifest -SUPPORTED_STORAGE_FRAMEWORKS = [ - "django.core.files.storage.FileSystemStorage", - "pulpcore.app.models.storage.FileSystem", -] - -pytestmark = pytest.mark.skipif( - settings.DEFAULT_FILE_STORAGE not in SUPPORTED_STORAGE_FRAMEWORKS, - reason="Cannot simulate bit-rot on this storage platform ({}).".format( - settings.DEFAULT_FILE_STORAGE - ), -) - - @pytest.fixture def repository_with_corrupted_artifacts( file_repository_api_client, @@ -42,19 +27,14 @@ def repository_with_corrupted_artifacts( # STEP 2: sample artifacts that will be modified on the filesystem later on content1, content2 = sample(get_files_in_manifest(remote.url), 2) - # Modify one artifact on disk. - artifact1_path = os.path.join( - settings.MEDIA_ROOT, artifacts_api_client.list(sha256=content1[1]).results[0].file - ) - with open(artifact1_path, "r+b") as f: + # Modify an artifact + artifact1_path = artifacts_api_client.list(sha256=content1[1]).results[0].file + with default_storage.open(artifact1_path, "w+b") as f: f.write(b"$a bit rot") - # Delete another one from disk. - artifact2_path = os.path.join( - settings.MEDIA_ROOT, artifacts_api_client.list(sha256=content2[1]).results[0].file - ) - os.remove(artifact2_path) - + # Delete an artifact + artifact2_path = artifacts_api_client.list(sha256=content2[1]).results[0].file + default_storage.delete(artifact2_path) return repo