From fde6653507a068467c6f463f3a9c46b5ce89f04b Mon Sep 17 00:00:00 2001 From: Jorrit Poelen Date: Fri, 8 Nov 2024 16:01:26 -0600 Subject: [PATCH] upgrade to globi-libs v0.27.0 --- pom.xml | 2 +- .../elton/cmd/CmdDatasets.java | 5 +- .../elton/cmd/CmdDefaultParams.java | 31 ++++++-- .../elton/cmd/CmdGet.java | 26 ++----- .../elton/cmd/CmdInteractions.java | 5 +- .../elton/cmd/CmdList.java | 3 +- .../elton/cmd/CmdLog.java | 4 +- .../elton/cmd/CmdNames.java | 3 +- .../elton/cmd/CmdNanoPubs.java | 4 +- .../elton/cmd/CmdReview.java | 9 ++- .../elton/cmd/CmdStream.java | 5 +- .../elton/cmd/CmdUpdate.java | 7 +- .../elton/cmd/CmdUtil.java | 72 ++++++++++++------- .../elton/cmd/StreamingDatasetsHandler.java | 16 ++++- .../store/CachePullThroughPrestonStore.java | 16 +++-- .../elton/util/DatasetRegistrySingleDir.java | 9 ++- .../elton/util/DatasetRegistryUtil.java | 53 +++++++++++--- .../CachePullThroughPrestonStoreTest.java | 6 +- 18 files changed, 188 insertions(+), 88 deletions(-) diff --git a/pom.xml b/pom.xml index 8f42ff7..488dbf1 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ Discover, access, and review, species interaction data - 0.26.6 + 0.27.0 0.7.8 1.7.30 4.6.3 diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdDatasets.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdDatasets.java index 16c0adb..fead814 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdDatasets.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdDatasets.java @@ -1,5 +1,6 @@ package org.globalbioticinteractions.elton.cmd; + import org.globalbioticinteractions.dataset.Dataset; import org.globalbioticinteractions.dataset.DatasetRegistry; import org.globalbioticinteractions.dataset.DatasetRegistryException; @@ -51,7 +52,9 @@ void run(PrintStream out) { DatasetRegistry registry = DatasetRegistryUtil.forCacheDirOrLocalDir( getCacheDir(), getWorkDir(), - createInputStreamFactory()); + createInputStreamFactory(), + getContentPathFactory(), + getProvenancePathFactory()); try { CmdUtil.handleNamespaces(registry, diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdDefaultParams.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdDefaultParams.java index edb3463..b029069 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdDefaultParams.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdDefaultParams.java @@ -1,5 +1,9 @@ package org.globalbioticinteractions.elton.cmd; +import org.globalbioticinteractions.cache.ContentPathFactory; +import org.globalbioticinteractions.cache.ContentPathFactoryDepth0; +import org.globalbioticinteractions.cache.ProvenancePathFactory; +import org.globalbioticinteractions.cache.ProvenancePathFactoryImpl; import org.globalbioticinteractions.elton.util.ProgressCursor; import org.globalbioticinteractions.elton.util.ProgressCursorFactory; import org.globalbioticinteractions.elton.util.ProgressCursorRotating; @@ -13,36 +17,41 @@ import java.util.List; abstract class CmdDefaultParams implements Runnable { - @CommandLine.Option (names = {"--cache-dir", "-c"}, + @CommandLine.Option(names = {"--cache-dir", "-c"}, description = "Cache directory" ) private String cacheDir = "./datasets"; - @CommandLine.Option (names = {"--work-dir", "-w"}, + @CommandLine.Option(names = {"--work-dir", "-w"}, description = "Work directory" ) private String workDir = "."; - @CommandLine.Option (names = {"--no-progress"}, + @CommandLine.Option(names = {"--no-progress"}, description = "Do not show progress indicator" ) private boolean noProgress = false; + public void setNamespaces(List namespaces) { this.namespaces = namespaces; } - @CommandLine.Parameters ( + @CommandLine.Parameters( description = "[namespace1] [namespace2] ..." ) private List namespaces = new ArrayList<>(); + private ContentPathFactory contentPathFactory = new ContentPathFactoryDepth0(); + private ProvenancePathFactory provenancePathFactory = new ProvenancePathFactoryImpl(); + private PrintStream stderr = System.err; private PrintStream stdout = System.out; private InputStream stdin = System.in; final private ProgressCursorFactory cursorFactory = new ProgressCursorFactory() { private final ProgressCursor cursor = new ProgressCursorRotating(stderr); + @Override public ProgressCursor createProgressCursor() { return cursor; @@ -92,14 +101,21 @@ public void setCacheDir(String cacheDir) { this.cacheDir = cacheDir; } + + public ContentPathFactory getContentPathFactory() { + return (this.contentPathFactory); + } + + public void setWorkDir(String workingDir) { this.workDir = workingDir; } public ProgressCursorFactory getProgressCursorFactory() { return noProgress - ? () -> () -> {} - : cursorFactory; + ? () -> () -> { + } + : cursorFactory; } public InputStreamFactoryLogging createInputStreamFactory() { @@ -107,4 +123,7 @@ public InputStreamFactoryLogging createInputStreamFactory() { } + public ProvenancePathFactory getProvenancePathFactory() { + return provenancePathFactory; + } } diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdGet.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdGet.java index 2888e26..d628d6f 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdGet.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdGet.java @@ -1,43 +1,23 @@ package org.globalbioticinteractions.elton.cmd; -import bio.guoda.preston.HashType; -import bio.guoda.preston.Hasher; -import bio.guoda.preston.RefNodeConstants; import bio.guoda.preston.RefNodeFactory; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.rdf.api.IRI; -import org.apache.commons.rdf.api.Quad; -import org.eol.globi.data.NodeFactory; -import org.eol.globi.service.CacheService; -import org.eol.globi.service.ResourceService; -import org.eol.globi.tool.NullImportLogger; -import org.globalbioticinteractions.dataset.Dataset; -import org.globalbioticinteractions.dataset.DatasetProxy; + import org.globalbioticinteractions.dataset.DatasetRegistry; import org.globalbioticinteractions.dataset.DatasetRegistryException; -import org.globalbioticinteractions.dataset.DatasetRegistryProxy; import org.globalbioticinteractions.elton.util.DatasetRegistryUtil; -import org.globalbioticinteractions.elton.util.NamespaceHandler; -import org.globalbioticinteractions.elton.util.NodeFactoryNull; -import org.nanopub.Run; import picocli.CommandLine; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintStream; -import java.net.URI; -import java.security.DigestInputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -61,7 +41,9 @@ void run(PrintStream out) { DatasetRegistry registry = DatasetRegistryUtil.forCacheDirOrLocalDir( getCacheDir(), getWorkDir(), - createInputStreamFactory() + createInputStreamFactory(), + getContentPathFactory(), + getProvenancePathFactory() ); final List actualNamespaces = new ArrayList<>(); diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdInteractions.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdInteractions.java index 4eafbdf..0705176 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdInteractions.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdInteractions.java @@ -3,6 +3,7 @@ import org.eol.globi.data.ImportLogger; import org.eol.globi.data.NodeFactory; import org.eol.globi.domain.LogContext; + import org.globalbioticinteractions.dataset.DatasetRegistry; import org.globalbioticinteractions.elton.util.DatasetRegistryUtil; import org.globalbioticinteractions.elton.util.ProgressUtil; @@ -29,7 +30,9 @@ void run(PrintStream out) { DatasetRegistry registry = DatasetRegistryUtil.forCacheDirOrLocalDir( getCacheDir(), getWorkDir(), - createInputStreamFactory()); + createInputStreamFactory(), + getContentPathFactory(), + getProvenancePathFactory()); NodeFactory nodeFactory = WriterUtil.nodeFactoryForInteractionWriting(!shouldSkipHeader(), out); diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdList.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdList.java index 62a1c73..0cc38ae 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdList.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdList.java @@ -1,6 +1,7 @@ package org.globalbioticinteractions.elton.cmd; import org.eol.globi.util.ResourceServiceRemote; + import org.globalbioticinteractions.dataset.DatasetRegistry; import org.globalbioticinteractions.dataset.DatasetRegistryException; import org.globalbioticinteractions.dataset.DatasetRegistryGitHubArchive; @@ -28,7 +29,7 @@ public void run() { public void run(PrintStream out) { InputStreamFactoryLogging inputStreamFactory = createInputStreamFactory(); - DatasetRegistry registryLocal = DatasetRegistryUtil.forCacheDirOrLocalDir(getCacheDir(), getWorkDir(), inputStreamFactory); + DatasetRegistry registryLocal = DatasetRegistryUtil.forCacheDirOrLocalDir(getCacheDir(), getWorkDir(), inputStreamFactory, getContentPathFactory(), getProvenancePathFactory()); File cacheDir = new File(getCacheDir()); List registries = diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdLog.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdLog.java index 2c0492c..31288c3 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdLog.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdLog.java @@ -9,6 +9,7 @@ import org.eol.globi.data.NodeFactory; import org.eol.globi.service.ResourceService; import org.eol.globi.tool.NullImportLogger; + import org.globalbioticinteractions.dataset.Dataset; import org.globalbioticinteractions.dataset.DatasetProxy; import org.globalbioticinteractions.dataset.DatasetRegistry; @@ -47,7 +48,8 @@ public void run() { DatasetRegistry registry = DatasetRegistryUtil.forCacheDirOrLocalDir( getCacheDir(), getWorkDir(), - createInputStreamFactory() + createInputStreamFactory(), + getContentPathFactory(), getProvenancePathFactory() ); IRI softwareAgent = RefNodeFactory.toIRI("https://zenodo.org/doi/10.5281/zenodo.998263"); diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdNames.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdNames.java index 5f235ad..8ddf570 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdNames.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdNames.java @@ -2,6 +2,7 @@ import org.eol.globi.data.NodeFactory; import org.eol.globi.tool.NullImportLogger; + import org.globalbioticinteractions.dataset.DatasetRegistry; import org.globalbioticinteractions.elton.util.DatasetRegistryUtil; import picocli.CommandLine; @@ -23,7 +24,7 @@ public void run() { void run(PrintStream out) { - DatasetRegistry registry = DatasetRegistryUtil.forCacheDirOrLocalDir(getCacheDir(), getWorkDir(), createInputStreamFactory()); + DatasetRegistry registry = DatasetRegistryUtil.forCacheDirOrLocalDir(getCacheDir(), getWorkDir(), createInputStreamFactory(), getContentPathFactory(), getProvenancePathFactory()); NodeFactory nodeFactory = WriterUtil.nodeFactoryForTaxonWriting(!shouldSkipHeader(), out); diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdNanoPubs.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdNanoPubs.java index a559aa2..f647f82 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdNanoPubs.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdNanoPubs.java @@ -1,6 +1,7 @@ package org.globalbioticinteractions.elton.cmd; import org.eol.globi.tool.NullImportLogger; + import org.globalbioticinteractions.dataset.DatasetRegistry; import org.globalbioticinteractions.elton.util.DatasetRegistryUtil; import org.globalbioticinteractions.elton.util.IdGenerator; @@ -43,7 +44,8 @@ void run(PrintStream out) { DatasetRegistry registry = DatasetRegistryUtil.forCacheDirOrLocalDir( getCacheDir(), getWorkDir(), - createInputStreamFactory()); + createInputStreamFactory(), + getContentPathFactory(), getProvenancePathFactory()); InteractionWriter serializer = createSerializer(out); diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdReview.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdReview.java index e287869..acaf6c6 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdReview.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdReview.java @@ -25,6 +25,7 @@ import org.eol.globi.util.InputStreamFactory; import org.eol.globi.util.ResourceServiceLocal; import org.eol.globi.util.ResourceServiceLocalAndRemote; + import org.globalbioticinteractions.dataset.CitationUtil; import org.globalbioticinteractions.dataset.Dataset; import org.globalbioticinteractions.dataset.DatasetConstant; @@ -87,6 +88,7 @@ public class CmdReview extends CmdTabularWriterParams { private String reviewId = UUID.randomUUID().toString(); + @Override public void run() { try { @@ -111,7 +113,8 @@ public void run() { DatasetRegistry registryLocal = DatasetRegistryUtil.forLocalDir( localNamespace, getCacheDir(), - new ResourceServiceLocalAndRemote(factory, new File(getCacheDir())) + new ResourceServiceLocalAndRemote(factory, new File(getCacheDir())), + getContentPathFactory() ); review(DatasetRegistryUtil.NAMESPACE_LOCAL, registryLocal, factory, shouldSkipHeader()); @@ -129,7 +132,9 @@ private void reviewCachedOrRemote(List namespaces, InputStreamFactory in review(namespace, DatasetRegistryUtil.forCacheDir( getCacheDir(), - new ResourceServiceLocal(inputStreamFactory) + new ResourceServiceLocal(inputStreamFactory), + getContentPathFactory(), + getProvenancePathFactory() ), inputStreamFactory, shouldSkipHeader() diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdStream.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdStream.java index 0908825..1509f99 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdStream.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdStream.java @@ -11,6 +11,7 @@ import org.eol.globi.data.NodeFactory; import org.eol.globi.domain.LogContext; import org.eol.globi.tool.NullImportLogger; + import org.globalbioticinteractions.elton.util.DatasetRegistryUtil; import org.globalbioticinteractions.elton.util.ProgressCursor; import org.globalbioticinteractions.elton.util.ProgressCursorFactory; @@ -69,7 +70,9 @@ public void run() { this.getStderr(), this.createInputStreamFactory(), new NodeFactoryFactoryImpl(shouldWriteHeader, recordType, loggerFactory.createImportLogger()), - loggerFactory + loggerFactory, + getContentPathFactory(), + getProvenancePathFactory() ); namespaceHandler.onNamespace(namespace); isFirst.set(false); diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdUpdate.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdUpdate.java index 9eb489a..d9f90ee 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdUpdate.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdUpdate.java @@ -3,6 +3,7 @@ import org.eol.globi.data.NodeFactory; import org.eol.globi.data.StudyImporterException; import org.eol.globi.util.DatasetImportUtil; + import org.globalbioticinteractions.dataset.Dataset; import org.globalbioticinteractions.dataset.DatasetFactory; import org.globalbioticinteractions.dataset.DatasetRegistry; @@ -65,11 +66,15 @@ public void run() { registryProxy, namespace, getCacheDir(), - inputStreamFactory); + inputStreamFactory, + getContentPathFactory(), + getProvenancePathFactory() + ); Dataset dataset = new DatasetFactory(registry, createInputStreamFactory()) .datasetFor(namespace); + NodeFactory factory = new NodeFactoryNull(); factory.getOrCreateDataset(dataset); try { diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdUtil.java b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdUtil.java index c901779..37de5eb 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/CmdUtil.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/CmdUtil.java @@ -1,8 +1,6 @@ package org.globalbioticinteractions.elton.cmd; -import bio.guoda.preston.process.StatementListener; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.rdf.api.Quad; import org.eol.globi.data.ImportLogger; import org.eol.globi.data.NodeFactory; import org.eol.globi.data.StudyImporterException; @@ -17,6 +15,9 @@ import org.globalbioticinteractions.cache.CacheFactory; import org.globalbioticinteractions.cache.CacheLocalReadonly; import org.globalbioticinteractions.cache.CacheProxy; +import org.globalbioticinteractions.cache.ContentPathFactory; +import org.globalbioticinteractions.cache.ProvenancePathFactory; + import org.globalbioticinteractions.dataset.Dataset; import org.globalbioticinteractions.dataset.DatasetFactory; import org.globalbioticinteractions.dataset.DatasetRegistry; @@ -62,25 +63,46 @@ static DatasetRegistry createDataFinderLoggingCaching( DatasetRegistry registry, String namespace, String cacheDir, - InputStreamFactory factory) { - CacheFactory cacheFactory = createCacheFactory(namespace, cacheDir, factory); + InputStreamFactory factory, + ContentPathFactory contentPathFactory, + ProvenancePathFactory provenancePathFactory) { + + CacheFactory cacheFactory = createCacheFactory( + namespace, + cacheDir, + factory, + contentPathFactory, + provenancePathFactory + ); return new DatasetRegistryWithCache(new DatasetRegistryLogger(registry, cacheDir), cacheFactory); } - public static CacheFactory createCacheFactory(String namespace, String cacheDir, InputStreamFactory factory) { + public static CacheFactory createCacheFactory(String namespace, + String cacheDir, + InputStreamFactory factory, + ContentPathFactory contentPathFactory, + ProvenancePathFactory provenancePathFactory) { return dataset -> { - ResourceService remote = new ResourceServiceLocalAndRemote(factory, new File(cacheDir)); - ResourceService local = new ResourceServiceLocal(factory); - Cache pullThroughCache = new CachePullThroughPrestonStore(namespace, cacheDir, remote, new StatementListener() { - - @Override - public void on(Quad quad) { + ResourceService remote = new ResourceServiceLocalAndRemote(factory, new File(cacheDir)); + ResourceService local = new ResourceServiceLocal(factory); + Cache pullThroughCache = new CachePullThroughPrestonStore( + namespace, + cacheDir, + remote, + quad -> { // ignore printing quads for now - } - }); - CacheLocalReadonly readOnlyCache = new CacheLocalReadonly(namespace, cacheDir, local); - return new CacheProxy(Arrays.asList(pullThroughCache, readOnlyCache)); - }; + }, + contentPathFactory); + + CacheLocalReadonly readOnlyCache = new CacheLocalReadonly( + namespace, + cacheDir, + local, + contentPathFactory, + provenancePathFactory + ); + return new CacheProxy(Arrays.asList(pullThroughCache, readOnlyCache)); + }; } public static List datasetInfo(Dataset dataset) { @@ -142,15 +164,15 @@ private static void handleSingleNamespace(DatasetRegistry registry, private static GeoNamesService createDummyGeoNamesService() { return new GeoNamesService() { - @Override - public boolean hasTermForLocale(String locality) { - return false; - } + @Override + public boolean hasTermForLocale(String locality) { + return false; + } - @Override - public LatLng findLatLng(String locality) throws IOException { - return null; - } - }; + @Override + public LatLng findLatLng(String locality) throws IOException { + return null; + } + }; } } diff --git a/src/main/java/org/globalbioticinteractions/elton/cmd/StreamingDatasetsHandler.java b/src/main/java/org/globalbioticinteractions/elton/cmd/StreamingDatasetsHandler.java index c23b5fa..bc1dd30 100644 --- a/src/main/java/org/globalbioticinteractions/elton/cmd/StreamingDatasetsHandler.java +++ b/src/main/java/org/globalbioticinteractions/elton/cmd/StreamingDatasetsHandler.java @@ -8,6 +8,9 @@ import org.eol.globi.util.InputStreamFactory; import org.globalbioticinteractions.cache.Cache; import org.globalbioticinteractions.cache.CacheFactory; +import org.globalbioticinteractions.cache.ContentPathFactory; +import org.globalbioticinteractions.cache.ProvenancePathFactory; + import org.globalbioticinteractions.dataset.Dataset; import org.globalbioticinteractions.dataset.DatasetWithCache; import org.globalbioticinteractions.dataset.DatasetWithResourceMapping; @@ -29,20 +32,25 @@ class StreamingDatasetsHandler implements NamespaceHandler { private final JsonNode config; private NodeFactorFactory nodeFactorFactory; private ImportLoggerFactory loggerFactory; + private ContentPathFactory contentPathFactory; + private ProvenancePathFactory provenancePathFactory; public StreamingDatasetsHandler(JsonNode config, String cacheDir, PrintStream stderr, InputStreamFactory inputStreamFactory, NodeFactorFactory nodeFactorFactory, - ImportLoggerFactory loggerFactory) { + ImportLoggerFactory loggerFactory, + ContentPathFactory contentPathFactory, + ProvenancePathFactory provenancePathFactory) { this.factory = inputStreamFactory; this.cacheDir = cacheDir; this.stderr = stderr; this.config = config; this.nodeFactorFactory = nodeFactorFactory; this.loggerFactory = loggerFactory; - + this.contentPathFactory = contentPathFactory; + this.provenancePathFactory = provenancePathFactory; } @Override @@ -51,7 +59,9 @@ public void onNamespace(String namespace) throws Exception { CacheFactory cacheFactory = CmdUtil.createCacheFactory( namespace, cacheDir, - factory + factory, + contentPathFactory, + provenancePathFactory ); Dataset dataset = new DatasetWithResourceMapping( diff --git a/src/main/java/org/globalbioticinteractions/elton/store/CachePullThroughPrestonStore.java b/src/main/java/org/globalbioticinteractions/elton/store/CachePullThroughPrestonStore.java index 60dee49..15572c1 100644 --- a/src/main/java/org/globalbioticinteractions/elton/store/CachePullThroughPrestonStore.java +++ b/src/main/java/org/globalbioticinteractions/elton/store/CachePullThroughPrestonStore.java @@ -17,6 +17,7 @@ import org.eol.globi.util.DateUtil; import org.globalbioticinteractions.cache.CachePullThrough; import org.globalbioticinteractions.cache.CacheUtil; +import org.globalbioticinteractions.cache.ContentPathFactory; import org.globalbioticinteractions.cache.ContentProvenance; import org.globalbioticinteractions.cache.ProvenanceLog; @@ -35,24 +36,29 @@ public class CachePullThroughPrestonStore extends CachePullThrough { public CachePullThroughPrestonStore( String namespace, String cachePath, - ResourceService resourceService + ResourceService resourceService, + ContentPathFactory contentPathFactory ) { this(namespace, cachePath, resourceService, new StatementListener() { @Override public void on(Quad quad) { // do nothing } - }); + }, contentPathFactory); } public CachePullThroughPrestonStore( String namespace, String cachePath, ResourceService resourceService, - StatementListener listener + StatementListener listener, + ContentPathFactory contentPathFactory ) { - super(namespace, cachePath, - resourceService); + super(namespace, + cachePath, + resourceService, + contentPathFactory + ); this.namespace = namespace; this.cachePath = cachePath; this.remote = resourceService; diff --git a/src/main/java/org/globalbioticinteractions/elton/util/DatasetRegistrySingleDir.java b/src/main/java/org/globalbioticinteractions/elton/util/DatasetRegistrySingleDir.java index 33c258e..32aeb44 100644 --- a/src/main/java/org/globalbioticinteractions/elton/util/DatasetRegistrySingleDir.java +++ b/src/main/java/org/globalbioticinteractions/elton/util/DatasetRegistrySingleDir.java @@ -1,6 +1,7 @@ package org.globalbioticinteractions.elton.util; import org.eol.globi.service.ResourceService; +import org.globalbioticinteractions.cache.ContentPathFactory; import org.globalbioticinteractions.dataset.Dataset; import org.globalbioticinteractions.dataset.DatasetImpl; import org.globalbioticinteractions.dataset.DatasetRegistry; @@ -16,13 +17,16 @@ public class DatasetRegistrySingleDir implements DatasetRegistry { private final URI localArchiveDir; private final String cacheDir; private ResourceService resourceService; + private ContentPathFactory contentPathFactory; public DatasetRegistrySingleDir(URI localArchiveDir, String cacheDir, - ResourceService resourceService) { + ResourceService resourceService, + ContentPathFactory contentPathFactory) { this.localArchiveDir = localArchiveDir; this.cacheDir = cacheDir; this.resourceService = resourceService; + this.contentPathFactory = contentPathFactory; } @Override @@ -47,7 +51,8 @@ public Dataset datasetFor(String namespace) throws DatasetRegistryException { new CachePullThroughPrestonStore( DatasetRegistryUtil.NAMESPACE_LOCAL, cacheDir, - this.resourceService + this.resourceService, + contentPathFactory ) ); } diff --git a/src/main/java/org/globalbioticinteractions/elton/util/DatasetRegistryUtil.java b/src/main/java/org/globalbioticinteractions/elton/util/DatasetRegistryUtil.java index 78a9d13..a4abeef 100644 --- a/src/main/java/org/globalbioticinteractions/elton/util/DatasetRegistryUtil.java +++ b/src/main/java/org/globalbioticinteractions/elton/util/DatasetRegistryUtil.java @@ -6,6 +6,9 @@ import org.eol.globi.util.ResourceServiceLocalAndRemote; import org.globalbioticinteractions.cache.CacheFactory; import org.globalbioticinteractions.cache.CacheLocalReadonly; +import org.globalbioticinteractions.cache.ContentPathFactory; +import org.globalbioticinteractions.cache.ProvenancePathFactory; + import org.globalbioticinteractions.dataset.DatasetRegistry; import org.globalbioticinteractions.dataset.DatasetRegistryException; import org.globalbioticinteractions.dataset.DatasetRegistryLocal; @@ -21,28 +24,36 @@ public class DatasetRegistryUtil { public static DatasetRegistry forLocalDir(final URI localArchiveDir, final String cacheDir, - ResourceService resourceServiceRemote) { + ResourceService resourceServiceRemote, + ContentPathFactory contentPathFactory) { return new DatasetRegistrySingleDir( localArchiveDir, cacheDir, - resourceServiceRemote + resourceServiceRemote, + contentPathFactory ); } private static CacheFactory getCacheFactoryLocal(String cacheDir, - ResourceService resourceServiceLocal) { + ResourceService resourceServiceLocal, + ContentPathFactory contentPathFactory, + ProvenancePathFactory provenancePathFactory) { return dataset -> new CacheLocalReadonly( dataset.getNamespace(), cacheDir, - resourceServiceLocal + resourceServiceLocal, + contentPathFactory, + provenancePathFactory ); } public static DatasetRegistry forCacheDir(String cacheDir, - ResourceService resourceServiceLocal) { + ResourceService resourceServiceLocal, + ContentPathFactory contentPathFactory, + ProvenancePathFactory provenancePathFactory) { return new DatasetRegistryLocal( cacheDir, - getCacheFactoryLocal(cacheDir, resourceServiceLocal), + getCacheFactoryLocal(cacheDir, resourceServiceLocal, contentPathFactory, provenancePathFactory), resourceServiceLocal); } @@ -54,17 +65,37 @@ private static boolean isEmpty(DatasetRegistry registry) { } } - public static DatasetRegistry forCacheDirOrLocalDir(String cacheDir, URI workDir, InputStreamFactory streamFactory) { - return forCacheDirOrLocalDir(cacheDir, workDir, new ResourceServiceLocal(streamFactory), new ResourceServiceLocalAndRemote(streamFactory, new File(cacheDir))); + public static DatasetRegistry forCacheDirOrLocalDir(String cacheDir, + URI workDir, + InputStreamFactory streamFactory, + ContentPathFactory contentPathFactory, + ProvenancePathFactory provenancePathFactory) { + return forCacheDirOrLocalDir(cacheDir, + workDir, + new ResourceServiceLocal(streamFactory), + new ResourceServiceLocalAndRemote(streamFactory, new File(cacheDir)), + contentPathFactory, + provenancePathFactory); } - public static DatasetRegistry forCacheDirOrLocalDir(String cacheDir, URI workDir, ResourceService resourceServiceLocal, ResourceService resourceServiceRemote) { - DatasetRegistry registry = forCacheDir(cacheDir, resourceServiceLocal); + public static DatasetRegistry forCacheDirOrLocalDir(String cacheDir, + URI workDir, + ResourceService resourceServiceLocal, + ResourceService resourceServiceRemote, + ContentPathFactory contentPathFactory, + ProvenancePathFactory provenancePathFactory) { + DatasetRegistry registry = forCacheDir( + cacheDir, + resourceServiceLocal, + contentPathFactory, + provenancePathFactory + ); if (isEmpty(registry)) { registry = forLocalDir( workDir, cacheDir, - resourceServiceRemote + resourceServiceRemote, + contentPathFactory ); } return registry; diff --git a/src/test/java/org/globalbioticinteractions/elton/store/CachePullThroughPrestonStoreTest.java b/src/test/java/org/globalbioticinteractions/elton/store/CachePullThroughPrestonStoreTest.java index e2fec9c..658df1f 100644 --- a/src/test/java/org/globalbioticinteractions/elton/store/CachePullThroughPrestonStoreTest.java +++ b/src/test/java/org/globalbioticinteractions/elton/store/CachePullThroughPrestonStoreTest.java @@ -10,13 +10,13 @@ import org.apache.commons.rdf.api.Quad; import org.eol.globi.util.ResourceServiceLocal; import org.globalbioticinteractions.cache.Cache; +import org.globalbioticinteractions.cache.ContentPathFactoryDepth0; import org.hamcrest.core.Is; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -42,7 +42,7 @@ public void testPrestonStore() throws IOException, URISyntaxException { "some/namespace" , folder.getRoot().getAbsolutePath() , new ResourceServiceLocal(in -> in) - , quads::add + , quads::add, new ContentPathFactoryDepth0() ); assertThat(quads.size(), Is.is(0)); @@ -91,7 +91,7 @@ public void testMissingFromPrestonStore() throws IOException, URISyntaxException Cache cache = new CachePullThroughPrestonStore( "some/namespace" , folder.getRoot().getAbsolutePath() - , new ResourceServiceLocal(in -> in) + , new ResourceServiceLocal(in -> in), new ContentPathFactoryDepth0() ); File namespaceDir = new File(folder.getRoot(), "some/namespace");