Skip to content

Commit

Permalink
Fixed repository version repair with S3
Browse files Browse the repository at this point in the history
This patch also adds repair API tests for S3 storage backend.

fixes: pulp#4776
fixes: pulp#4806
  • Loading branch information
dkliban committed Nov 30, 2023
1 parent 701bbc4 commit eb09b6d
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 14 deletions.
2 changes: 2 additions & 0 deletions CHANGES/4776.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fixed a bug where the repository version repair operation would accidently try to look for a file
in the default domain.
2 changes: 2 additions & 0 deletions CHANGES/4806.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fixed a bug where the repository version repair operation would always report a digest mismatch when
using S3 storage.
8 changes: 5 additions & 3 deletions pulpcore/app/tasks/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,16 +96,18 @@ async def _repair_ca(content_artifact, repaired=None):


def _verify_artifact(artifact):
domain = get_domain()
storage = domain.get_storage()
try:
# verify files digest
hasher = hashlib.sha256()
with artifact.file as fp:
with storage.open(artifact.file.name) as fp:
for chunk in fp.chunks(CHUNK_SIZE):
hasher.update(chunk)
return hasher.hexdigest() == artifact.sha256
except FileNotFoundError:
return False

return hasher.hexdigest() == artifact.sha256


async def _repair_artifacts_for_content(subset=None, verify_checksums=True):
loop = asyncio.get_event_loop()
Expand Down
19 changes: 8 additions & 11 deletions pulpcore/tests/functional/api/using_plugin/test_repair.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import pytest
import os

from django.core.files.storage import default_storage
from random import sample

from pulpcore.client.pulpcore import Repair
Expand All @@ -14,6 +15,7 @@
SUPPORTED_STORAGE_FRAMEWORKS = [
"django.core.files.storage.FileSystemStorage",
"pulpcore.app.models.storage.FileSystem",
"storages.backends.s3boto3.S3Boto3Storage",
]

pytestmark = pytest.mark.skipif(
Expand Down Expand Up @@ -42,19 +44,14 @@ def repository_with_corrupted_artifacts(
# STEP 2: sample artifacts that will be modified on the filesystem later on
content1, content2 = sample(get_files_in_manifest(remote.url), 2)

# Modify one artifact on disk.
artifact1_path = os.path.join(
settings.MEDIA_ROOT, artifacts_api_client.list(sha256=content1[1]).results[0].file
)
with open(artifact1_path, "r+b") as f:
# Modify and artifact
artifact1_path = artifacts_api_client.list(sha256=content1[1]).results[0].file
with default_storage.open(artifact1_path, "w+b") as f:
f.write(b"$a bit rot")

# Delete another one from disk.
artifact2_path = os.path.join(
settings.MEDIA_ROOT, artifacts_api_client.list(sha256=content2[1]).results[0].file
)
os.remove(artifact2_path)

# Delete an artifact
artifact2_path = artifacts_api_client.list(sha256=content2[1]).results[0].file
default_storage.delete(artifact2_path)
return repo


Expand Down

0 comments on commit eb09b6d

Please sign in to comment.