Enable listing of zip files in fromfiles for index, gather, search

src/python/tests/test_index.py

@@ -123,6 +123,32 @@ def test_index_manifest(runtmp, capfd):
     assert 'index is done' in runtmp.last_result.err
 
 
+def test_index_multiple_manifests(runtmp, capfd):
+    # test index with text file of multiple manifests
+    sig2 = get_test_data('2.fa.sig.gz')
+    sig47 = get_test_data('47.fa.sig.gz')
+    sig63 = get_test_data('63.fa.sig.gz')
+    sigs = [sig2, sig47, sig63]
+    manifests = []
+    for sig in sigs:
+        sig_mf = runtmp.output(os.path.basename(sig) + ".mf.zip")
+        runtmp.sourmash("sig", "cat", sig, "-o", sig_mf)
+        manifests.append(sig_mf)
+
+    # assert False
+    manifests_zips_list = runtmp.output('manifest_zips.txt')
+    make_file_list(manifests_zips_list, manifests)
+
+    output = runtmp.output('out.db')
+    runtmp.sourmash('scripts', 'index', manifests_zips_list,
+                        '-o', output)
+
+    captured = capfd.readouterr()
+    print(captured.err)
+    print(runtmp.last_result.err)
+    assert 'index is done' in runtmp.last_result.err
+
+
 def test_index_bad_siglist_2(runtmp, capfd):
     # test with a bad siglist (containing a missing file)
     against_list = runtmp.output('against.txt')

src/utils.rs

@@ -542,6 +542,46 @@ pub fn collection_from_zipfile(sigpath: &Path, report_type: &ReportType) -> Resu
     }
 }
 
+// Make a collection from anything except a pathlist, as this is called
+// from collection_from_pathlist
+fn collection_from_zipfile_or_signature_or_manifest(
+    sigpath: &Path,
+    report_type: &ReportType,
+) -> Result<Option<Collection>, Option<anyhow::Error>> {
+// returns collection and number of failures
+    let mut last_error = None;
+    let collection = if sigpath.extension().map_or(false, |ext| ext == "zip") {
+        match collection_from_zipfile(&sigpath, &report_type) {
+            Ok(coll) => Some(coll),
+            Err(e) => {
+                last_error = Some(e);
+                None
+            }
+        }
+    } else {
+        None
+    };
+
+    let collection =
+        collection.or_else(|| match collection_from_manifest(&sigpath, &report_type) {
+            Ok(coll) => Some(coll),
+            Err(e) => {
+                last_error = Some(e);
+                None
+            }
+        });
+
+    let collection =
+        collection.or_else(|| match collection_from_signature(&sigpath, &report_type) {
+            Ok(coll) => Some(coll),
+            Err(e) => {
+                last_error = Some(e);
+                None
+            }
+        });
+    Ok(collection)
+}
+
 fn collection_from_manifest(
     sigpath: &Path,
     report_type: &ReportType,
@@ -585,58 +625,50 @@ fn collection_from_pathlist(
         )
     })?;
     let reader = BufReader::new(file);
+    let mut last_error: std::option::Option<anyhow::Error> = None;
 
     // load list of paths
     let lines: Vec<_> = reader
         .lines()
         .filter_map(|line| match line {
-            Ok(path) => Some(path),
+            Ok(path) => Some(PathBuf::from(path)),
             Err(_err) => None,
         })
         .collect();
 
     // load sketches from paths in parallel.
     let n_failed = AtomicUsize::new(0);
-    let records: Vec<Record> = lines
+
+    // Load all entries as collections
+    let record_paths = lines
         .par_iter()
-        .filter_map(|path| match Signature::from_path(path) {
-            Ok(signatures) => {
-                let recs: Vec<Record> = signatures
-                    .into_iter()
-                    .flat_map(|v| Record::from_sig(&v, path))
-                    .collect();
-                Some(recs)
-            }
+        .filter_map(|path| match collection_from_zipfile_or_signature_or_manifest(&path, &report_type) {
+            Ok(collection) => {
+                // For each record in the collection, get its path filename
+                Some(collection
+                    .unwrap()
+                    .manifest()
+                    .iter()
+                    .filter_map(|record| match record.internal_location(){
+                        PathBuf => Some(location)
+
+                    })
+                    .collect())
+            },
             Err(err) => {
-                eprintln!("Sketch loading error: {}", err);
                 eprintln!("WARNING: could not load sketches from path '{}'", path);
                 let _ = n_failed.fetch_add(1, atomic::Ordering::SeqCst);
                 None
             }
-        })
-        .flatten()
-        .collect();
+        }).flatten().collect::<Vec<PathBuf>>();
 
-    if records.is_empty() {
-        eprintln!(
-            "No valid signatures found in {} pathlist '{}'",
-            report_type, sigpath
-        );
-    }
 
-    let manifest: Manifest = records.into();
-    let collection = Collection::new(
-        manifest,
-        InnerStorage::new(
-            FSStorage::builder()
-                .fullpath("".into())
-                .subdir("".into())
-                .build(),
-        ),
-    );
+    // Now load the path filenames as one big collection
+    let collection = Collection::from_paths(&record_paths);
+
     let n_failed = n_failed.load(atomic::Ordering::SeqCst);
 
-    Ok((collection, n_failed))
+    Ok((collection?, n_failed))
 }
 
 fn collection_from_signature(sigpath: &Path, report_type: &ReportType) -> Result<Collection> {