Skip to content

Commit

Permalink
Tweaks and nits
Browse files Browse the repository at this point in the history
  • Loading branch information
dbutenhof committed Feb 20, 2024
1 parent aa01ac0 commit 9ccd500
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 16 deletions.
2 changes: 1 addition & 1 deletion lib/pbench/cli/server/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,7 @@ def report(
cache_m = CacheManager(config, logger)
verifier.status("starting discovery")
watcher.update("discovering cache")
cache_m.full_discovery(full=False)
cache_m.full_discovery(search=False)
watcher.update("processing reports")
verifier.status("finished discovery")
if all or archive:
Expand Down
29 changes: 15 additions & 14 deletions lib/pbench/server/cache_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -827,8 +827,8 @@ def get_contents(self, path: str, origin: str) -> JSONOBJECT:
file_list = []
for f in artifact.iterdir():
relative = f.relative_to(self.unpacked)
append_to = file_list
if f.is_symlink():
append_to = file_list
target = f.resolve()
try:
link = target.relative_to(self.unpacked)
Expand Down Expand Up @@ -1267,16 +1267,17 @@ def _add_if_tarball(self, file: Path, md5: Optional[str] = None) -> bool:
Returns:
true if the tarball was added
"""
if file.is_file() and Dataset.is_tarball(file):
hash = md5 if md5 else get_tarball_md5(file)
tarball = Tarball(file, hash, self)
self.tarballs[tarball.name] = tarball
self.datasets[tarball.resource_id] = tarball
tarball.check_unpacked()
return True
else:

if not (file.is_file() and Dataset.is_tarball(file)):
return False

hash = md5 if md5 else get_tarball_md5(file)
tarball = Tarball(file, hash, self)
self.tarballs[tarball.name] = tarball
self.datasets[tarball.resource_id] = tarball
tarball.check_unpacked()
return True

def _discover_tarballs(self):
"""Discover the known tarballs
Expand Down Expand Up @@ -1431,19 +1432,19 @@ def __init__(self, options: PbenchServerConfig, logger: Logger):
# resource_id.
self.datasets: dict[str, Tarball] = {}

def full_discovery(self, full: bool = True) -> "CacheManager":
def full_discovery(self, search: bool = True) -> "CacheManager":
"""Discover the ARCHIVE and CACHE trees
Full discovery is only needed for reporting, and is not required
to find, create, or delete a specific dataset. (See find_dataset.)
Args:
full: search the ARCHIVE tree rather than using Dataset table
search: search the ARCHIVE tree rather than using the Dataset table
Returns:
CacheManager instance for chaining
"""
if full:
if search:
self._discover_controllers()
else:
self._discover_datasets()
Expand Down Expand Up @@ -1540,12 +1541,12 @@ def _discover_datasets(self):
Metadata,
and_(Dataset.id == Metadata.dataset_ref, Metadata.key == "server"),
)
.yield_per(500)
.yield_per(1000)
.all()
)

for name, resource_id, path in rows:
if not all((resource_id, path)):
if not path:
# This runs asychronously with normal operation, and we might
# find a dataset before the "server.tarball-path" metadata is
# set. Issue a warning in case this becomes a concern, but
Expand Down
2 changes: 1 addition & 1 deletion lib/pbench/test/unit/server/test_cache_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -2016,7 +2016,7 @@ def test_discover_datasets(
ds = Dataset(name=name, resource_id=md5, owner=drb, access="public")
ds.add()
Metadata.setvalue(ds, Metadata.TARBALL_PATH, path)
cm2 = CacheManager(server_config, make_logger).full_discovery(full=False)
cm2 = CacheManager(server_config, make_logger).full_discovery(search=False)
assert cm2[md5].name == name

def test_lockref(self, monkeypatch):
Expand Down

0 comments on commit 9ccd500

Please sign in to comment.