Merge pull request #1281 from Sage-Bionetworks/develop-fix-getfilesin…

…dataset

[bug fix] Added includesType parameter when using walk function

schematic/store/synapse.py

@@ -35,7 +35,7 @@
 from synapseclient.table import CsvFileTable, build_table, Schema
 from synapseclient.annotations import from_synapse_annotations
 from synapseclient.core.exceptions import SynapseHTTPError, SynapseAuthenticationError, SynapseUnmetAccessRestrictions
-from synapseutils import walk
+import synapseutils
 from synapseutils.copy_functions import changeFileMetaData
 
 import uuid
@@ -413,7 +413,7 @@ def getFilesInStorageDataset(
         """
 
         # select all files within a given storage dataset folder (top level folder in a Synapse storage project or folder marked with contentType = 'dataset')
-        walked_path = walk(self.syn, datasetId)
+        walked_path = synapseutils.walk(self.syn, datasetId, includeTypes=["folder", "file"])
 
         file_list = []
 

tests/test_store.py

@@ -1,21 +1,25 @@
 from __future__ import annotations
-import os
-import math
+
 import logging
-import pytest
-from time import sleep 
-from tenacity import Retrying, RetryError, stop_after_attempt, wait_random_exponential
+import math
+import os
+from time import sleep
+from unittest.mock import patch
 
 import pandas as pd
+import pytest
 from synapseclient import EntityViewSchema, Folder
-
-from schematic.models.metadata import MetadataModel
-from schematic.store.base import BaseStorage
-from schematic.store.synapse import SynapseStorage, DatasetFileView, ManifestDownload
-from schematic.schemas.generator import SchemaGenerator
 from synapseclient.core.exceptions import SynapseHTTPError
 from synapseclient.entity import File
+from tenacity import (RetryError, Retrying, stop_after_attempt,
+                      wait_random_exponential)
+
 from schematic.configuration.configuration import Configuration
+from schematic.models.metadata import MetadataModel
+from schematic.schemas.generator import SchemaGenerator
+from schematic.store.base import BaseStorage
+from schematic.store.synapse import (DatasetFileView, ManifestDownload,
+                                     SynapseStorage)
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
@@ -232,6 +236,24 @@ def test_getDatasetProject(self, dataset_id, synapse_store):
 
         with pytest.raises(PermissionError):
             synapse_store.getDatasetProject("syn12345678")
+
+    @pytest.mark.parametrize("full_path,expected", [(True, [('syn126', 'parent_folder/test_file'), ('syn125', 'parent_folder/test_folder/test_file_2')]),(False, [('syn126', 'test_file'), ('syn125', 'test_file_2')])])
+    def test_getFilesInStorageDataset(self, synapse_store, full_path, expected):
+        mock_return = [
+        (
+            ("parent_folder", "syn123"),
+            [("test_folder", "syn124")],
+            [("test_file", "syn126")],
+        ),
+        (
+            (os.path.join("parent_folder", "test_folder"), "syn124"),
+            [],
+            [("test_file_2", "syn125")],
+        ),
+        ]
+        with patch('synapseutils.walk_functions._helpWalk', return_value=mock_return):
+            file_list = synapse_store.getFilesInStorageDataset(datasetId="syn_mock", fileNames=None, fullpath=full_path)
+            assert file_list == expected
 
     @pytest.mark.parametrize("downloadFile", [True, False])
     def test_getDatasetManifest(self, synapse_store, downloadFile):