From ed8e0affe9b663d9d17245fabd7509cd3d805b7f Mon Sep 17 00:00:00 2001 From: Sidharth Hulyalkar Date: Wed, 16 Aug 2023 17:40:30 -0500 Subject: [PATCH 1/5] Adds funtionality: if a tiff_list is specified in ops, .nd2 files to only be pulled from the list --- suite2p/io/utils.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/suite2p/io/utils.py b/suite2p/io/utils.py index 22d0c321..cbe0fee8 100644 --- a/suite2p/io/utils.py +++ b/suite2p/io/utils.py @@ -167,21 +167,29 @@ def get_nd2_list(ops): if ops['look_one_level_down'], then all nd2's in all folders + one level down """ froot = ops['data_path'] - fold_list = ops['data_path'] - fsall = [] - nfs = 0 - first_tiffs = [] - for k,fld in enumerate(fold_list): - fs, ftiffs = list_files(fld, ops['look_one_level_down'], - ["*.nd2"]) - fsall.extend(fs) - first_tiffs.extend(list(ftiffs)) - if len(fs)==0: - print('Could not find any nd2 files') - raise Exception('no nd2s') - else: - ops['first_tiffs'] = np.array(first_tiffs).astype('bool') + if 'tiff_list' in ops: + fsall = [] + for tif in ops['tiff_list']: + fsall.append(os.path.join(froot[0], tif)) + ops['first_tiffs'] = np.zeros((len(fsall),), dtype='bool') + ops['first_tiffs'][0] = True print('** Found %d nd2 files - converting to binary **'%(len(fsall))) + else: + fold_list = ops['data_path'] + fsall = [] + nfs = 0 + first_tiffs = [] + for k,fld in enumerate(fold_list): + fs, ftiffs = list_files(fld, ops['look_one_level_down'], + ["*.nd2"]) + fsall.extend(fs) + first_tiffs.extend(list(ftiffs)) + if len(fs)==0: + print('Could not find any nd2 files') + raise Exception('no nd2s') + else: + ops['first_tiffs'] = np.array(first_tiffs).astype('bool') + print('** Found %d nd2 files - converting to binary **'%(len(fsall))) return fsall, ops From 755433c6007babd962ba59d329579db026012f9e Mon Sep 17 00:00:00 2001 From: Sidharth Hulyalkar Date: Wed, 16 Aug 2023 17:45:19 -0500 Subject: [PATCH 2/5] update get_nd2_list comment --- suite2p/io/utils.py | 293 +++++++++++++++++++++++--------------------- 1 file changed, 150 insertions(+), 143 deletions(-) diff --git a/suite2p/io/utils.py b/suite2p/io/utils.py index cbe0fee8..850cae4e 100644 --- a/suite2p/io/utils.py +++ b/suite2p/io/utils.py @@ -6,59 +6,61 @@ from natsort import natsorted -def search_for_ext(rootdir, extension = 'tif', look_one_level_down=False): +def search_for_ext(rootdir, extension="tif", look_one_level_down=False): filepaths = [] if os.path.isdir(rootdir): # search root dir - tmp = glob.glob(os.path.join(rootdir,'*.'+extension)) + tmp = glob.glob(os.path.join(rootdir, "*." + extension)) if len(tmp): filepaths.extend([t for t in natsorted(tmp)]) # search one level down if look_one_level_down: dirs = natsorted(os.listdir(rootdir)) for d in dirs: - if os.path.isdir(os.path.join(rootdir,d)): - tmp = glob.glob(os.path.join(rootdir, d, '*.'+extension)) + if os.path.isdir(os.path.join(rootdir, d)): + tmp = glob.glob(os.path.join(rootdir, d, "*." + extension)) if len(tmp): filepaths.extend([t for t in natsorted(tmp)]) if len(filepaths): return filepaths else: - raise OSError('Could not find files, check path [{0}]'.format(rootdir)) + raise OSError("Could not find files, check path [{0}]".format(rootdir)) + def get_sbx_list(ops): - """ make list of scanbox files to process + """make list of scanbox files to process if ops['subfolders'], then all tiffs ops['data_path'][0] / ops['subfolders'] / *.sbx if ops['look_one_level_down'], then all tiffs in all folders + one level down TODO: Implement "tiff_list" functionality """ - froot = ops['data_path'] + froot = ops["data_path"] # use a user-specified list of tiffs - if len(froot)==1: - if 'subfolders' in ops and len(ops['subfolders'])>0: + if len(froot) == 1: + if "subfolders" in ops and len(ops["subfolders"]) > 0: fold_list = [] - for folder_down in ops['subfolders']: + for folder_down in ops["subfolders"]: fold = os.path.join(froot[0], folder_down) fold_list.append(fold) else: - fold_list = ops['data_path'] + fold_list = ops["data_path"] else: fold_list = froot fsall = [] - for k,fld in enumerate(fold_list): - fs = search_for_ext(fld, - extension = 'sbx', - look_one_level_down = ops['look_one_level_down']) + for k, fld in enumerate(fold_list): + fs = search_for_ext( + fld, extension="sbx", look_one_level_down=ops["look_one_level_down"] + ) fsall.extend(fs) - if len(fsall)==0: + if len(fsall) == 0: print(fold_list) - raise Exception('No files, check path.') + raise Exception("No files, check path.") else: - print('** Found %d sbx - converting to binary **'%(len(fsall))) + print("** Found %d sbx - converting to binary **" % (len(fsall))) return fsall, ops + def list_h5(ops): - froot = os.path.dirname(ops['h5py']) + froot = os.path.dirname(ops["h5py"]) lpath = os.path.join(froot, "*.h5") fs = natsorted(glob.glob(lpath)) lpath = os.path.join(froot, "*.hdf5") @@ -66,19 +68,19 @@ def list_h5(ops): fs.extend(fs2) return fs + def list_files(froot, look_one_level_down, exts): - """ get list of files with exts in folder froot + one level down maybe - """ + """get list of files with exts in folder froot + one level down maybe""" fs = [] for e in exts: lpath = os.path.join(froot, e) fs.extend(glob.glob(lpath)) fs = natsorted(set(fs)) if len(fs) > 0: - first_tiffs = np.zeros((len(fs),), 'bool') + first_tiffs = np.zeros((len(fs),), "bool") first_tiffs[0] = True else: - first_tiffs = np.zeros(0, 'bool') + first_tiffs = np.zeros(0, "bool") lfs = len(fs) if look_one_level_down: fdir = natsorted(glob.glob(os.path.join(froot, "*/"))) @@ -90,111 +92,113 @@ def list_files(froot, look_one_level_down, exts): fsnew = natsorted(set(fsnew)) if len(fsnew) > 0: fs.extend(fsnew) - first_tiffs = np.append(first_tiffs, np.zeros((len(fsnew),), 'bool')) + first_tiffs = np.append(first_tiffs, np.zeros((len(fsnew),), "bool")) first_tiffs[lfs] = True lfs = len(fs) return fs, first_tiffs + def get_h5_list(ops): - """ make list of h5 files to process + """make list of h5 files to process if ops['look_one_level_down'], then all h5's in all folders + one level down """ - froot = ops['data_path'] - fold_list = ops['data_path'] + froot = ops["data_path"] + fold_list = ops["data_path"] fsall = [] nfs = 0 first_tiffs = [] - for k,fld in enumerate(fold_list): - fs, ftiffs = list_files(fld, ops['look_one_level_down'], - ["*.h5", "*.hdf5"]) + for k, fld in enumerate(fold_list): + fs, ftiffs = list_files(fld, ops["look_one_level_down"], ["*.h5", "*.hdf5"]) fsall.extend(fs) first_tiffs.extend(list(ftiffs)) - if len(fs)==0: - print('Could not find any h5 files') - raise Exception('no h5s') + if len(fs) == 0: + print("Could not find any h5 files") + raise Exception("no h5s") else: - ops['first_tiffs'] = np.array(first_tiffs).astype('bool') - print('** Found %d h5 files - converting to binary **'%(len(fsall))) - #print('Found %d tifs'%(len(fsall))) + ops["first_tiffs"] = np.array(first_tiffs).astype("bool") + print("** Found %d h5 files - converting to binary **" % (len(fsall))) + # print('Found %d tifs'%(len(fsall))) return fsall, ops def get_tif_list(ops): - """ make list of tiffs to process + """make list of tiffs to process if ops['subfolders'], then all tiffs ops['data_path'][0] / ops['subfolders'] / *.tif if ops['look_one_level_down'], then all tiffs in all folders + one level down if ops['tiff_list'], then ops['data_path'][0] / ops['tiff_list'] ONLY """ - froot = ops['data_path'] + froot = ops["data_path"] # use a user-specified list of tiffs - if 'tiff_list' in ops: + if "tiff_list" in ops: fsall = [] - for tif in ops['tiff_list']: + for tif in ops["tiff_list"]: fsall.append(os.path.join(froot[0], tif)) - ops['first_tiffs'] = np.zeros((len(fsall),), dtype='bool') - ops['first_tiffs'][0] = True - print('** Found %d tifs - converting to binary **'%(len(fsall))) + ops["first_tiffs"] = np.zeros((len(fsall),), dtype="bool") + ops["first_tiffs"][0] = True + print("** Found %d tifs - converting to binary **" % (len(fsall))) else: - if len(froot)==1: - if 'subfolders' in ops and len(ops['subfolders'])>0: + if len(froot) == 1: + if "subfolders" in ops and len(ops["subfolders"]) > 0: fold_list = [] - for folder_down in ops['subfolders']: + for folder_down in ops["subfolders"]: fold = os.path.join(froot[0], folder_down) fold_list.append(fold) else: - fold_list = ops['data_path'] + fold_list = ops["data_path"] else: fold_list = froot fsall = [] nfs = 0 first_tiffs = [] - for k,fld in enumerate(fold_list): - fs, ftiffs = list_files(fld, ops['look_one_level_down'], - ["*.tif", "*.tiff", "*.TIF", "*.TIFF"]) + for k, fld in enumerate(fold_list): + fs, ftiffs = list_files( + fld, ops["look_one_level_down"], ["*.tif", "*.tiff", "*.TIF", "*.TIFF"] + ) fsall.extend(fs) first_tiffs.extend(list(ftiffs)) - if len(fsall)==0: - print('Could not find any tiffs') - raise Exception('no tiffs') + if len(fsall) == 0: + print("Could not find any tiffs") + raise Exception("no tiffs") else: - ops['first_tiffs'] = np.array(first_tiffs).astype('bool') - print('** Found %d tifs - converting to binary **'%(len(fsall))) + ops["first_tiffs"] = np.array(first_tiffs).astype("bool") + print("** Found %d tifs - converting to binary **" % (len(fsall))) return fsall, ops def get_nd2_list(ops): - """ make list of nd2 files to process - if ops['look_one_level_down'], then all nd2's in all folders + one level down """ - froot = ops['data_path'] - if 'tiff_list' in ops: + if tiff_list is specified in ops, construct list of .nd2 files specified + else construct a list of all nd2 files to process + - if ops['look_one_level_down'], then obtain all nd2's in all folders + one level down + """ + froot = ops["data_path"] + if "tiff_list" in ops: fsall = [] - for tif in ops['tiff_list']: + for tif in ops["tiff_list"]: fsall.append(os.path.join(froot[0], tif)) - ops['first_tiffs'] = np.zeros((len(fsall),), dtype='bool') - ops['first_tiffs'][0] = True - print('** Found %d nd2 files - converting to binary **'%(len(fsall))) + ops["first_tiffs"] = np.zeros((len(fsall),), dtype="bool") + ops["first_tiffs"][0] = True + print("** Found %d nd2 files - converting to binary **" % (len(fsall))) else: - fold_list = ops['data_path'] + fold_list = ops["data_path"] fsall = [] nfs = 0 first_tiffs = [] - for k,fld in enumerate(fold_list): - fs, ftiffs = list_files(fld, ops['look_one_level_down'], - ["*.nd2"]) + for k, fld in enumerate(fold_list): + fs, ftiffs = list_files(fld, ops["look_one_level_down"], ["*.nd2"]) fsall.extend(fs) first_tiffs.extend(list(ftiffs)) - if len(fs)==0: - print('Could not find any nd2 files') - raise Exception('no nd2s') + if len(fs) == 0: + print("Could not find any nd2 files") + raise Exception("no nd2s") else: - ops['first_tiffs'] = np.array(first_tiffs).astype('bool') - print('** Found %d nd2 files - converting to binary **'%(len(fsall))) + ops["first_tiffs"] = np.array(first_tiffs).astype("bool") + print("** Found %d nd2 files - converting to binary **" % (len(fsall))) return fsall, ops def find_files_open_binaries(ops1, ish5=False): - """ finds tiffs or h5 files and opens binaries for writing + """finds tiffs or h5 files and opens binaries for writing Parameters ---------- @@ -209,63 +213,62 @@ def find_files_open_binaries(ops1, ish5=False): """ reg_file = [] - reg_file_chan2=[] - + reg_file_chan2 = [] for ops in ops1: - nchannels = ops['nchannels'] - if 'keep_movie_raw' in ops and ops['keep_movie_raw']: - reg_file.append(open(ops['raw_file'], 'wb')) - if nchannels>1: - reg_file_chan2.append(open(ops['raw_file_chan2'], 'wb')) + nchannels = ops["nchannels"] + if "keep_movie_raw" in ops and ops["keep_movie_raw"]: + reg_file.append(open(ops["raw_file"], "wb")) + if nchannels > 1: + reg_file_chan2.append(open(ops["raw_file_chan2"], "wb")) else: - reg_file.append(open(ops['reg_file'], 'wb')) - if nchannels>1: - reg_file_chan2.append(open(ops['reg_file_chan2'], 'wb')) + reg_file.append(open(ops["reg_file"], "wb")) + if nchannels > 1: + reg_file_chan2.append(open(ops["reg_file_chan2"], "wb")) - if 'input_format' in ops.keys(): - input_format = ops['input_format'] + if "input_format" in ops.keys(): + input_format = ops["input_format"] else: - input_format = 'tif' + input_format = "tif" if ish5: - input_format = 'h5' + input_format = "h5" print(input_format) - if input_format == 'h5': - if len(ops1[0]['data_path'])>0: + if input_format == "h5": + if len(ops1[0]["data_path"]) > 0: fs, ops2 = get_h5_list(ops1[0]) - print('NOTE: using a list of h5 files:') + print("NOTE: using a list of h5 files:") print(fs) # find h5's else: - if ops1[0]['look_one_level_down']: + if ops1[0]["look_one_level_down"]: fs = list_h5(ops1[0]) - print('NOTE: using a list of h5 files:') + print("NOTE: using a list of h5 files:") print(fs) else: - fs = [ops1[0]['h5py']] - elif input_format == 'sbx': + fs = [ops1[0]["h5py"]] + elif input_format == "sbx": # find sbx fs, ops2 = get_sbx_list(ops1[0]) - print('Scanbox files:') - print('\n'.join(fs)) - elif input_format == 'nd2': + print("Scanbox files:") + print("\n".join(fs)) + elif input_format == "nd2": # find nd2s fs, ops2 = get_nd2_list(ops1[0]) - print('Nikon files:') - print('\n'.join(fs)) + print("Nikon files:") + print("\n".join(fs)) else: # find tiffs fs, ops2 = get_tif_list(ops1[0]) for ops in ops1: - ops['first_tiffs'] = ops2['first_tiffs'] - ops['frames_per_folder'] = np.zeros((ops2['first_tiffs'].sum(),), np.int32) + ops["first_tiffs"] = ops2["first_tiffs"] + ops["frames_per_folder"] = np.zeros((ops2["first_tiffs"].sum(),), np.int32) for ops in ops1: - ops['filelist'] = fs + ops["filelist"] = fs return ops1, fs, reg_file, reg_file_chan2 def init_ops(ops): - """ initializes ops files for each plane in recording + """initializes ops files for each plane in recording Parameters ---------- @@ -281,50 +284,54 @@ def init_ops(ops): """ - nplanes = ops['nplanes'] - nchannels = ops['nchannels'] - if 'lines' in ops: - lines = ops['lines'] - if 'iplane' in ops: - iplane = ops['iplane'] - #ops['nplanes'] = len(ops['lines']) + nplanes = ops["nplanes"] + nchannels = ops["nchannels"] + if "lines" in ops: + lines = ops["lines"] + if "iplane" in ops: + iplane = ops["iplane"] + # ops['nplanes'] = len(ops['lines']) ops1 = [] - if ('fast_disk' not in ops) or len(ops['fast_disk'])==0: - ops['fast_disk'] = ops['save_path0'] - fast_disk = ops['fast_disk'] + if ("fast_disk" not in ops) or len(ops["fast_disk"]) == 0: + ops["fast_disk"] = ops["save_path0"] + fast_disk = ops["fast_disk"] # for mesoscope recording FOV locations - if 'dy' in ops and ops['dy']!='': - dy = ops['dy'] - dx = ops['dx'] + if "dy" in ops and ops["dy"] != "": + dy = ops["dy"] + dx = ops["dx"] # compile ops into list across planes - for j in range(0,nplanes): - if len(ops['save_folder']) > 0: - ops['save_path'] = os.path.join(ops['save_path0'], ops['save_folder'], 'plane%d'%j) + for j in range(0, nplanes): + if len(ops["save_folder"]) > 0: + ops["save_path"] = os.path.join( + ops["save_path0"], ops["save_folder"], "plane%d" % j + ) else: - ops['save_path'] = os.path.join(ops['save_path0'], 'suite2p', 'plane%d'%j) - - if ('fast_disk' not in ops) or len(ops['fast_disk'])==0: - ops['fast_disk'] = ops['save_path0'] - ops['fast_disk'] = os.path.join(fast_disk, 'suite2p', 'plane%d'%j) - ops['ops_path'] = os.path.join(ops['save_path'],'ops.npy') - ops['reg_file'] = os.path.join(ops['fast_disk'], 'data.bin') - if 'keep_movie_raw' in ops and ops['keep_movie_raw']: - ops['raw_file'] = os.path.join(ops['fast_disk'], 'data_raw.bin') - if 'lines' in ops: - ops['lines'] = lines[j] - if 'iplane' in ops: - ops['iplane'] = iplane[j] - if nchannels>1: - ops['reg_file_chan2'] = os.path.join(ops['fast_disk'], 'data_chan2.bin') - if 'keep_movie_raw' in ops and ops['keep_movie_raw']: - ops['raw_file_chan2'] = os.path.join(ops['fast_disk'], 'data_chan2_raw.bin') - if 'dy' in ops and ops['dy']!='': - ops['dy'] = dy[j] - ops['dx'] = dx[j] - if not os.path.isdir(ops['fast_disk']): - os.makedirs(ops['fast_disk']) - if not os.path.isdir(ops['save_path']): - os.makedirs(ops['save_path']) + ops["save_path"] = os.path.join(ops["save_path0"], "suite2p", "plane%d" % j) + + if ("fast_disk" not in ops) or len(ops["fast_disk"]) == 0: + ops["fast_disk"] = ops["save_path0"] + ops["fast_disk"] = os.path.join(fast_disk, "suite2p", "plane%d" % j) + ops["ops_path"] = os.path.join(ops["save_path"], "ops.npy") + ops["reg_file"] = os.path.join(ops["fast_disk"], "data.bin") + if "keep_movie_raw" in ops and ops["keep_movie_raw"]: + ops["raw_file"] = os.path.join(ops["fast_disk"], "data_raw.bin") + if "lines" in ops: + ops["lines"] = lines[j] + if "iplane" in ops: + ops["iplane"] = iplane[j] + if nchannels > 1: + ops["reg_file_chan2"] = os.path.join(ops["fast_disk"], "data_chan2.bin") + if "keep_movie_raw" in ops and ops["keep_movie_raw"]: + ops["raw_file_chan2"] = os.path.join( + ops["fast_disk"], "data_chan2_raw.bin" + ) + if "dy" in ops and ops["dy"] != "": + ops["dy"] = dy[j] + ops["dx"] = dx[j] + if not os.path.isdir(ops["fast_disk"]): + os.makedirs(ops["fast_disk"]) + if not os.path.isdir(ops["save_path"]): + os.makedirs(ops["save_path"]) ops1.append(ops.copy()) return ops1 From 9d0c70807419b7f0e19c1550203131fbb4d56589 Mon Sep 17 00:00:00 2001 From: Sidharth Hulyalkar Date: Wed, 16 Aug 2023 17:56:31 -0500 Subject: [PATCH 3/5] move print outside if else --- suite2p/io/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/suite2p/io/utils.py b/suite2p/io/utils.py index 850cae4e..38e2e071 100644 --- a/suite2p/io/utils.py +++ b/suite2p/io/utils.py @@ -172,16 +172,14 @@ def get_nd2_list(ops): - if ops['look_one_level_down'], then obtain all nd2's in all folders + one level down """ froot = ops["data_path"] + fsall = [] if "tiff_list" in ops: - fsall = [] for tif in ops["tiff_list"]: fsall.append(os.path.join(froot[0], tif)) ops["first_tiffs"] = np.zeros((len(fsall),), dtype="bool") ops["first_tiffs"][0] = True - print("** Found %d nd2 files - converting to binary **" % (len(fsall))) else: fold_list = ops["data_path"] - fsall = [] nfs = 0 first_tiffs = [] for k, fld in enumerate(fold_list): @@ -193,7 +191,7 @@ def get_nd2_list(ops): raise Exception("no nd2s") else: ops["first_tiffs"] = np.array(first_tiffs).astype("bool") - print("** Found %d nd2 files - converting to binary **" % (len(fsall))) + print("** Found %d nd2 files - converting to binary **" % (len(fsall))) return fsall, ops From 05c4e4ccb7dcd736648fef6b3d6e160419d166fc Mon Sep 17 00:00:00 2001 From: Sidharth Hulyalkar Date: Wed, 16 Aug 2023 17:58:03 -0500 Subject: [PATCH 4/5] Update suite2p/io/utils.py Co-authored-by: Thinh Nguyen --- suite2p/io/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/suite2p/io/utils.py b/suite2p/io/utils.py index 38e2e071..1ff46a01 100644 --- a/suite2p/io/utils.py +++ b/suite2p/io/utils.py @@ -174,8 +174,8 @@ def get_nd2_list(ops): froot = ops["data_path"] fsall = [] if "tiff_list" in ops: - for tif in ops["tiff_list"]: - fsall.append(os.path.join(froot[0], tif)) + for nd2_file in ops["tiff_list"]: + fsall.append(os.path.join(froot[0], nd2_file)) ops["first_tiffs"] = np.zeros((len(fsall),), dtype="bool") ops["first_tiffs"][0] = True else: From 8221197893f693e29cd38e6a9ad752be29124bbe Mon Sep 17 00:00:00 2001 From: Sidharth Hulyalkar Date: Wed, 16 Aug 2023 18:07:41 -0500 Subject: [PATCH 5/5] Update suite2p/io/utils.py Co-authored-by: Thinh Nguyen --- suite2p/io/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suite2p/io/utils.py b/suite2p/io/utils.py index 1ff46a01..79a804f4 100644 --- a/suite2p/io/utils.py +++ b/suite2p/io/utils.py @@ -186,7 +186,7 @@ def get_nd2_list(ops): fs, ftiffs = list_files(fld, ops["look_one_level_down"], ["*.nd2"]) fsall.extend(fs) first_tiffs.extend(list(ftiffs)) - if len(fs) == 0: + if len(fsall) == 0: print("Could not find any nd2 files") raise Exception("no nd2s") else: