diff --git a/README.md b/README.md index 5a96753..18cd3bd 100644 --- a/README.md +++ b/README.md @@ -219,9 +219,9 @@ NGSQSTTS015A0 (20211026111452695-847006) NGSQSTTS015A0 2021-10-26T09:14:53.14381 ## `download` ```txt -Usage: postman-cli download [-hV] [-o=] -u= [-s=[, - ...]]... (--password: - env= | --password: +Usage: postman-cli download [-hV] [--ignore-subdirectories] [-o=] + -u= [-s=[,...]]... + (--password:env= | --password: prop= | --password) (-f= | SAMPLE_IDENTIFIER...) @@ -246,17 +246,36 @@ Options: (case-insensitive) suffixes -o, --output-dir= specify where to write the downloaded data + --ignore-subdirectories + put all files into one directory regardless of the + directory structure on the server; conflicts + with files with equal names are not addressed -h, --help Show this help message and exit. -V, --version Print version information and exit. Optional: specify a config file by running postman with '@/path/to/config.txt'. -A detailed documentation can be found at -https://github.com/qbicsoftware/postman-cli#readme. +A detailed documentation can be found at https://github.com/qbicsoftware/postman-cli#readme. ``` The `download` command allows you to download data from our storage to your machine. Use the `--output-dir` option to specify a location on your client the location will be interpreted relative to your working directory. +By using the `--ignore-subdirectories` flag, you signal qpostman that you are not interested in the directory structure on the server. +qpostman will thus put all files it finds into the same top-level folder. + +**default download behaviour** +```text +my/awesome/path/file1.fastq.gz -> QABCD/my/awesome/path/file1.fastq.gz +my/awesome/other/path/file2.fastq.gz -> QABCD/my/awesome/other/path/file2.fastq.gz +my/awesome/additional/path/file3.fastq.gz -> QABCD/my/awesome/additional/path/file3.fastq.gz +``` +**with `--ignore-subdirectories`** +```text +with --ignore-subdirectories +my/awesome/path/file1.fastq.gz -> QABCD/file1.fastq.gz +my/awesome/other/path/file2.fastq.gz -> QABCD/file2.fastq.gz +my/awesome/additional/path/file3.fastq.gz -> QABCD/file3.fastq.gz +``` ##### File integrity check Postman computes the CRC32 checksum for all input streams using the native Java utility class [CRC32](https://docs.oracle.com/javase/8/docs/api/java/util/zip/CRC32.html). Postman favours [`CheckedInputStream`](https://docs.oracle.com/javase/7/docs/api/java/util/zip/CheckedInputStream.html) over the traditional InputStream, and promotes the CRC checksum computation. diff --git a/src/main/java/life/qbic/qpostman/common/functions/FindSourceSample.java b/src/main/java/life/qbic/qpostman/common/functions/FindSourceSample.java index 18232bc..fce456e 100644 --- a/src/main/java/life/qbic/qpostman/common/functions/FindSourceSample.java +++ b/src/main/java/life/qbic/qpostman/common/functions/FindSourceSample.java @@ -3,6 +3,7 @@ import static java.util.Objects.requireNonNull; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import java.util.List; import java.util.Optional; import java.util.function.Function; @@ -23,10 +24,11 @@ private Optional findSourceSample(Sample sample) { if (sample.getType().getCode().equals(sourceSampleTypeCode)) { return Optional.of(sample); } - if (sample.getParents().isEmpty()) { + List parents = sample.getParents(); + if (parents.isEmpty()) { return Optional.empty(); } - for (Sample parent : sample.getParents()) { + for (Sample parent : parents) { Optional sourceSample = findSourceSample(parent); if (sourceSample.isPresent()) { return sourceSample; diff --git a/src/main/java/life/qbic/qpostman/common/functions/SearchDataSets.java b/src/main/java/life/qbic/qpostman/common/functions/SearchDataSets.java index caae664..266b107 100644 --- a/src/main/java/life/qbic/qpostman/common/functions/SearchDataSets.java +++ b/src/main/java/life/qbic/qpostman/common/functions/SearchDataSets.java @@ -39,6 +39,7 @@ private Collection searchDataSets(Collection userInput) } private Stream searchSamples(SampleQuery sampleQuery) { + return applicationServerApi.searchSamples(OpenBisSessionProvider.get().getToken(), sampleQuery.searchCriteria(), sampleQuery.fetchOptions()).getObjects().stream(); } @@ -58,15 +59,28 @@ public SampleSearchCriteria searchCriteria() { } public SampleFetchOptions fetchOptions() { + // for all samples fetch the direct parents with type SampleFetchOptions parentFetchOptions = new SampleFetchOptions(); parentFetchOptions.withType(); + // fetch dataset with sample, sample type and parents with type, propagate fetch options to children. SampleFetchOptions sampleFetchOptions = new SampleFetchOptions(); sampleFetchOptions.withDataSets().withSample(); sampleFetchOptions.withType(); sampleFetchOptions.withParents(); sampleFetchOptions.withParentsUsing(parentFetchOptions); sampleFetchOptions.withChildrenUsing(sampleFetchOptions); - return sampleFetchOptions; + // for the root sample, fetch all parents recursively + SampleFetchOptions rootParentFetchOptions = new SampleFetchOptions(); + rootParentFetchOptions.withType(); + rootParentFetchOptions.withParentsUsing(rootParentFetchOptions); + // use same fetch options as for all other samples + fetching all parents recursively instead of direct parents. + SampleFetchOptions rootSampleFetchOptions = new SampleFetchOptions(); + rootSampleFetchOptions.withDataSets().withSample(); + rootSampleFetchOptions.withType(); + rootSampleFetchOptions.withParents(); + rootSampleFetchOptions.withChildrenUsing(sampleFetchOptions); + rootSampleFetchOptions.withParentsUsing(rootParentFetchOptions); + return rootSampleFetchOptions; } } diff --git a/src/main/java/life/qbic/qpostman/download/DownloadCommand.java b/src/main/java/life/qbic/qpostman/download/DownloadCommand.java index be1cb9e..353282e 100644 --- a/src/main/java/life/qbic/qpostman/download/DownloadCommand.java +++ b/src/main/java/life/qbic/qpostman/download/DownloadCommand.java @@ -124,7 +124,8 @@ private Functions functions() { SearchDataSets searchDataSets = new SearchDataSets(applicationServerApi); FileFilter myAwesomeFileFilter = FileFilter.create().withSuffixes(filterOptions.suffixes); WriteFileToDisk writeFileToDisk = new WriteFileToDisk(dataStoreServerApis().toArray(IDataStoreServerApi[]::new)[0], - downloadOptions.bufferSize, Path.of(downloadOptions.outputPath), downloadOptions.successiveDownloadAttempts); + downloadOptions.bufferSize, Path.of(downloadOptions.outputPath), downloadOptions.successiveDownloadAttempts, + downloadOptions.ignoreSubDirectories); FindSourceSample findSourceSample = new FindSourceSample(serverOptions.sourceSampleType); SortFiles sortFiles = new SortFiles(); DataSetWrapper.setFindSourceFunction(findSourceSample); diff --git a/src/main/java/life/qbic/qpostman/download/DownloadOptions.java b/src/main/java/life/qbic/qpostman/download/DownloadOptions.java index 675bdfa..b91320e 100644 --- a/src/main/java/life/qbic/qpostman/download/DownloadOptions.java +++ b/src/main/java/life/qbic/qpostman/download/DownloadOptions.java @@ -5,6 +5,7 @@ import java.util.Optional; import java.util.StringJoiner; +import picocli.CommandLine.Help.Visibility; public class DownloadOptions { @Option(names = {"--buffer-size"}, @@ -26,6 +27,11 @@ public class DownloadOptions { hidden = true) public int successiveDownloadAttempts; + @Option(names = "--ignore-subdirectories", defaultValue = "false", + description = "put all files into one directory regardless of the directory structure on the server; conflicts with files with equal names are not addressed", + showDefaultValue = Visibility.ON_DEMAND) + public boolean ignoreSubDirectories; + @Override public String toString() { return new StringJoiner(", ", DownloadOptions.class.getSimpleName() + "[", "]") diff --git a/src/main/java/life/qbic/qpostman/download/WriteFileToDisk.java b/src/main/java/life/qbic/qpostman/download/WriteFileToDisk.java index ca89b51..445e697 100644 --- a/src/main/java/life/qbic/qpostman/download/WriteFileToDisk.java +++ b/src/main/java/life/qbic/qpostman/download/WriteFileToDisk.java @@ -31,18 +31,22 @@ public class WriteFileToDisk implements Function { private final Path outputDirectory; private final int downloadAttempts; + private final boolean ignoreDirectories; + private static final Logger log = LogManager.getLogger(WriteFileToDisk.class); public WriteFileToDisk(IDataStoreServerApi dataStoreServerApi, int bufferSize, Path outputDirectory, - int downloadAttempts) { + int downloadAttempts, boolean ignoreDirectories) { this.dataStoreServerApi = dataStoreServerApi; this.bufferSize = bufferSize; this.outputDirectory = outputDirectory; this.downloadAttempts = downloadAttempts; + this.ignoreDirectories = ignoreDirectories; } - private static Path toOutputPath(DataFile dataFile, Path outputDirectory) { + private Path toOutputPath(DataFile dataFile, Path outputDirectory) { Path dataSetDirectory = outputDirectory.resolve(dataFile.dataSet().sampleCode()); - return dataSetDirectory.resolve(dataFile.filePath()); + String fileSpecific = ignoreDirectories ? dataFile.fileName() :dataFile.filePath(); + return dataSetDirectory.resolve(fileSpecific); } private AutoClosableDataSetFileDownloadReader toReader(DataFile dataFile) { @@ -64,9 +68,9 @@ private InputStream toInputStream(DataSetFileDownloadReader reader) { */ @Override public DownloadReport apply(DataFile dataFile) { - int bufferSize = - (dataFile.fileSize().bytes() < this.bufferSize) ? (int) dataFile.fileSize().bytes() - : this.bufferSize; + int bufferSize = (dataFile.fileSize().bytes() < this.bufferSize) + ? (int) dataFile.fileSize().bytes() + : this.bufferSize; Path outputPath = toOutputPath(dataFile, outputDirectory); if (WriteUtils.doesExistWithCrc32(outputPath, dataFile.crc32(), bufferSize)) { log.info("File " + outputPath + " exists on your machine.");