Add --ignore-subdirectories flag (#169)

* Add option `--ignore-directories`

* make sure parents are fetched

README.md

@@ -219,9 +219,9 @@ NGSQSTTS015A0 (20211026111452695-847006)	NGSQSTTS015A0	2021-10-26T09:14:53.14381
 ## `download`
 
 ```txt
-Usage: postman-cli download [-hV] [-o=<outputPath>] -u=<user> [-s=<suffix>[,
-                            <suffix>...]]... (--password:
-                            env=<environment-variable> | --password:
+Usage: postman-cli download [-hV] [--ignore-subdirectories] [-o=<outputPath>]
+                            -u=<user> [-s=<suffix>[,<suffix>...]]...
+                            (--password:env=<environment-variable> | --password:
                             prop=<system-property> | --password) (-f=<filePath>
                             | SAMPLE_IDENTIFIER...)
 
@@ -246,17 +246,36 @@ Options:
                                (case-insensitive) suffixes
   -o, --output-dir=<outputPath>
                              specify where to write the downloaded data
+      --ignore-subdirectories
+                             put all files into one directory regardless of the
+                               directory structure on the server; conflicts
+                               with files with equal names are not addressed
   -h, --help                 Show this help message and exit.
   -V, --version              Print version information and exit.
 
 Optional: specify a config file by running postman with '@/path/to/config.txt'.
-A detailed documentation can be found at
-https://github.com/qbicsoftware/postman-cli#readme.
+A detailed documentation can be found at https://github.com/qbicsoftware/postman-cli#readme.
 ```
 The `download` command allows you to download data from our storage to your machine. 
 
 Use the `--output-dir` option to specify a location on your client the location will be interpreted relative to your working directory.
 
+By using the `--ignore-subdirectories` flag, you signal qpostman that you are not interested in the directory structure on the server.
+qpostman will thus put all files it finds into the same top-level folder.
+
+**default download behaviour**
+```text
+my/awesome/path/file1.fastq.gz            -> QABCD/my/awesome/path/file1.fastq.gz
+my/awesome/other/path/file2.fastq.gz      -> QABCD/my/awesome/other/path/file2.fastq.gz
+my/awesome/additional/path/file3.fastq.gz -> QABCD/my/awesome/additional/path/file3.fastq.gz
+```
+**with `--ignore-subdirectories`**
+```text
+with --ignore-subdirectories
+my/awesome/path/file1.fastq.gz            -> QABCD/file1.fastq.gz
+my/awesome/other/path/file2.fastq.gz      -> QABCD/file2.fastq.gz
+my/awesome/additional/path/file3.fastq.gz -> QABCD/file3.fastq.gz
+```
 ##### File integrity check
 Postman computes the CRC32 checksum for all input streams using the native Java utility class [CRC32](https://docs.oracle.com/javase/8/docs/api/java/util/zip/CRC32.html). Postman favours [`CheckedInputStream`](https://docs.oracle.com/javase/7/docs/api/java/util/zip/CheckedInputStream.html)
 over the traditional InputStream, and promotes the CRC checksum computation.

src/main/java/life/qbic/qpostman/common/functions/FindSourceSample.java

@@ -3,6 +3,7 @@
 import static java.util.Objects.requireNonNull;
 
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
+import java.util.List;
 import java.util.Optional;
 import java.util.function.Function;
 
@@ -23,10 +24,11 @@ private Optional<Sample> findSourceSample(Sample sample) {
         if (sample.getType().getCode().equals(sourceSampleTypeCode)) {
             return Optional.of(sample);
         }
-        if (sample.getParents().isEmpty()) {
+        List<Sample> parents = sample.getParents();
+        if (parents.isEmpty()) {
             return Optional.empty();
         }
-        for (Sample parent : sample.getParents()) {
+        for (Sample parent : parents) {
             Optional<Sample> sourceSample = findSourceSample(parent);
             if (sourceSample.isPresent()) {
                 return sourceSample;

src/main/java/life/qbic/qpostman/common/functions/SearchDataSets.java

@@ -39,6 +39,7 @@ private Collection<DataSetWrapper> searchDataSets(Collection<String> userInput)
     }
 
     private Stream<Sample> searchSamples(SampleQuery sampleQuery) {
+
         return applicationServerApi.searchSamples(OpenBisSessionProvider.get().getToken(), sampleQuery.searchCriteria(), sampleQuery.fetchOptions()).getObjects().stream();
     }
 
@@ -58,15 +59,28 @@ public SampleSearchCriteria searchCriteria() {
         }
 
         public SampleFetchOptions fetchOptions() {
+            // for all samples fetch the direct parents with type
             SampleFetchOptions parentFetchOptions = new SampleFetchOptions();
             parentFetchOptions.withType();
+            // fetch dataset with sample, sample type and parents with type, propagate fetch options to children.
             SampleFetchOptions sampleFetchOptions = new SampleFetchOptions();
             sampleFetchOptions.withDataSets().withSample();
             sampleFetchOptions.withType();
             sampleFetchOptions.withParents();
             sampleFetchOptions.withParentsUsing(parentFetchOptions);
             sampleFetchOptions.withChildrenUsing(sampleFetchOptions);
-            return sampleFetchOptions;
+            // for the root sample, fetch all parents recursively
+            SampleFetchOptions rootParentFetchOptions = new SampleFetchOptions();
+            rootParentFetchOptions.withType();
+            rootParentFetchOptions.withParentsUsing(rootParentFetchOptions);
+            // use same fetch options as for all other samples + fetching all parents recursively instead of direct parents.
+            SampleFetchOptions rootSampleFetchOptions = new SampleFetchOptions();
+            rootSampleFetchOptions.withDataSets().withSample();
+            rootSampleFetchOptions.withType();
+            rootSampleFetchOptions.withParents();
+            rootSampleFetchOptions.withChildrenUsing(sampleFetchOptions);
+            rootSampleFetchOptions.withParentsUsing(rootParentFetchOptions);
+            return rootSampleFetchOptions;
         }
     }
 

src/main/java/life/qbic/qpostman/download/DownloadCommand.java

@@ -124,7 +124,8 @@ private Functions functions() {
         SearchDataSets searchDataSets = new SearchDataSets(applicationServerApi);
         FileFilter myAwesomeFileFilter = FileFilter.create().withSuffixes(filterOptions.suffixes);
         WriteFileToDisk writeFileToDisk = new WriteFileToDisk(dataStoreServerApis().toArray(IDataStoreServerApi[]::new)[0],
-            downloadOptions.bufferSize, Path.of(downloadOptions.outputPath), downloadOptions.successiveDownloadAttempts);
+            downloadOptions.bufferSize, Path.of(downloadOptions.outputPath), downloadOptions.successiveDownloadAttempts,
+            downloadOptions.ignoreSubDirectories);
         FindSourceSample findSourceSample = new FindSourceSample(serverOptions.sourceSampleType);
         SortFiles sortFiles = new SortFiles();
         DataSetWrapper.setFindSourceFunction(findSourceSample);

src/main/java/life/qbic/qpostman/download/DownloadOptions.java

@@ -5,6 +5,7 @@
 
 import java.util.Optional;
 import java.util.StringJoiner;
+import picocli.CommandLine.Help.Visibility;
 
 public class DownloadOptions {
     @Option(names = {"--buffer-size"},
@@ -26,6 +27,11 @@ public class DownloadOptions {
         hidden = true)
     public int successiveDownloadAttempts;
 
+    @Option(names = "--ignore-subdirectories", defaultValue = "false",
+        description = "put all files into one directory regardless of the directory structure on the server; conflicts with files with equal names are not addressed",
+        showDefaultValue = Visibility.ON_DEMAND)
+    public boolean ignoreSubDirectories;
+
     @Override
     public String toString() {
         return new StringJoiner(", ", DownloadOptions.class.getSimpleName() + "[", "]")

src/main/java/life/qbic/qpostman/download/WriteFileToDisk.java

@@ -31,18 +31,22 @@ public class WriteFileToDisk implements Function<DataFile, DownloadReport> {
     private final Path outputDirectory;
     private final int downloadAttempts;
 
+    private final boolean ignoreDirectories;
+
     private static final Logger log = LogManager.getLogger(WriteFileToDisk.class);
     public WriteFileToDisk(IDataStoreServerApi dataStoreServerApi, int bufferSize, Path outputDirectory,
-        int downloadAttempts) {
+        int downloadAttempts, boolean ignoreDirectories) {
         this.dataStoreServerApi = dataStoreServerApi;
         this.bufferSize = bufferSize;
         this.outputDirectory = outputDirectory;
         this.downloadAttempts = downloadAttempts;
+        this.ignoreDirectories = ignoreDirectories;
     }
 
-    private static Path toOutputPath(DataFile dataFile, Path outputDirectory) {
+    private Path toOutputPath(DataFile dataFile, Path outputDirectory) {
         Path dataSetDirectory = outputDirectory.resolve(dataFile.dataSet().sampleCode());
-        return dataSetDirectory.resolve(dataFile.filePath());
+        String fileSpecific = ignoreDirectories ? dataFile.fileName() :dataFile.filePath();
+      return dataSetDirectory.resolve(fileSpecific);
     }
 
     private AutoClosableDataSetFileDownloadReader toReader(DataFile dataFile) {
@@ -64,9 +68,9 @@ private InputStream toInputStream(DataSetFileDownloadReader reader) {
      */
     @Override
     public DownloadReport apply(DataFile dataFile) {
-        int bufferSize =
-            (dataFile.fileSize().bytes() < this.bufferSize) ? (int) dataFile.fileSize().bytes()
-                : this.bufferSize;
+        int bufferSize = (dataFile.fileSize().bytes() < this.bufferSize)
+            ? (int) dataFile.fileSize().bytes()
+            : this.bufferSize;
         Path outputPath = toOutputPath(dataFile, outputDirectory);
         if (WriteUtils.doesExistWithCrc32(outputPath, dataFile.crc32(), bufferSize)) {
             log.info("File " + outputPath + " exists on your machine.");