From f26f134bfe617a2ef8b6308f990cb3b0f93e3e44 Mon Sep 17 00:00:00 2001 From: Abhimat Local Date: Thu, 28 Sep 2023 16:56:45 -0700 Subject: [PATCH 1/6] Initial support for running stf on main and submaps in parallel --- kai/reduce/analysis.py | 157 +++++++++++++++++++++++++++++++---------- 1 file changed, 119 insertions(+), 38 deletions(-) diff --git a/kai/reduce/analysis.py b/kai/reduce/analysis.py index 4d0187e..5ff0946 100644 --- a/kai/reduce/analysis.py +++ b/kai/reduce/analysis.py @@ -13,6 +13,26 @@ import subprocess import pylab as py import pdb +from multiprocessing import Pool + +def idl_process_run(batch_file): + """ + Function to help run parallel processes for IDL, used for StarFinder + + Parameters + ---------- + batch_file : str + Path of the bath file containing the IDL command to run + """ + + batch_file_log = batch_file + '.log' + + cmd = 'idl < ' + batch_file + ' >& ' + batch_file_log + #os.system(cmd) + subp = subprocess.Popen(cmd, shell=True, executable="/bin/tcsh") + tmp = subp.communicate() + + return class Analysis(object): """ @@ -26,7 +46,8 @@ def __init__(self, epoch, rootDir='/g/lu/data/orion/', filt='kp', combo_stf_dir = None, epochDirSuffix=None, imgSuffix=None, stfDir=None, useDistorted=False, cleanList='c.lis', - airopa_mode='single', stf_version=None, + airopa_mode='single', stf_version=None, stf_debug=False, + stf_parallelize=False, instrument=instruments.default_inst): """ Set up Analysis object. @@ -56,6 +77,12 @@ def __init__(self, epoch, rootDir='/g/lu/data/orion/', filt='kp', (e.g.: 'v3_1') instrument : instruments object, optional Instrument of data. Default is `instruments.default_inst` + stf_debug : boolean, default=False + Keyword to specify if starfinder should be run in debug mode + stf_parallelize : boolean, default=True + Keyword to specify if starfinder runs on main map and sub maps + should be run in parallel. + By default, runs in parallel (all 4 run together). """ # Setup default parameters self.type = 'ao' @@ -66,6 +93,8 @@ def __init__(self, epoch, rootDir='/g/lu/data/orion/', filt='kp', self.airopa_mode = airopa_mode self.trimfake = 1 self.stfFlags = '' + self.stf_debug = stf_debug + self.stf_parallelize = stf_parallelize self.starlist = rootDir + 'source_list/psf_central.dat' self.labellist = rootDir+ 'source_list/label.dat' @@ -251,40 +280,92 @@ def starfinderCombo(self, oldPsf=False): os.chdir(self.dirComboStf) if self.type == 'ao': - # Write an IDL batch file - fileIDLbatch = 'idlbatch_combo_' + self.filt - fileIDLlog = fileIDLbatch + '.log' - util.rmall([fileIDLlog, fileIDLbatch]) - - _batch = open(fileIDLbatch, 'w') - _batch.write("find_stf_new, ") - _batch.write("'" + self.epoch + "', ") - _batch.write("'" + self.filt + "', ") - _batch.write("corr_main=%3.1f, " % self.corrMain) - _batch.write("corr_subs=%3.1f, " % self.corrSub) - # Only send in the deblend flag if using it! - if self.deblend != None: - _batch.write("deblend=" + str(self.deblend) + ", ") - _batch.write("cooStar='" + self.cooStar + "', ") - _batch.write("suffixEpoch='" + self.suffix + "', ") - _batch.write("imgSuffix='" + self.imgSuffix + "', ") - _batch.write("starlist='" + self.starlist + "', ") - if self.airopa_mode == 'legacy': - _batch.write("/legacy, ") - if self.airopa_mode == 'variable': - _batch.write("/aoopt, ") - _batch.write("trimfake=" + str(self.trimfake) + ", ") - if not oldPsf: - _batch.write("/makePsf, ") + # Write an IDL batch file for the main map and each submap + batch_files = [] + batch_file_logs = [] - _batch.write("rootDir='" + self.rootDir + "'") - - # Support for arbitrary starfinder flags. - _batch.write(self.stfFlags) + combos = ['main', '1', '2', '3'] + + for cur_combo in combos: + batch_file = 'idlbatch_main_' + self.filt + batch_file_log = batch_file + '.log' + + if cur_combo != 'main': + batch_file = 'idlbatch_sm{0}_{1}'.format(cur_combo, self.filt) + batch_file_log = batch_file + '.log' + + util.rmall([batch_file, batch_file_log]) + + batch_out = "" + + combo_file_path = "{0}/mag{1}_{2}.fits".format( + self.dirCombo, self.epoch, self.filt, + ) + + if cur_combo != 'main': + combo_file_path = "{0}/m{1}_{2}_{3}.fits".format( + self.dirCombo, self.epoch, self.filt, cur_combo, + ) + + batch_out += "find_stf, " + batch_out += "'{0}', ".format(combo_file_path) + + if cur_combo == 'main': + batch_out += "{0:3.1f}, ".format(self.corrMain) + else: + batch_out += "{0:3.1f}, ".format(self.corrSub) + + batch_out += "ttStar='TTstar', gsStar='', " + + # Add deblend flag if using it + if self.deblend != None: + batch_out += "deblend={0}, ".format(self.deblend) + + if not oldPsf: + batch_out += "/makePsf, " + + batch_out += "makeRes=1, " + batch_out += "makeStars=1, " + + if self.airopa_mode == 'legacy': + batch_out += "/legacy, " + + if self.airopa_mode == 'variable': + batch_out += "/aoopt, " + + batch_out += "cooStar='{0}', ".format(self.cooStar) + batch_out += "starlist='{0}', ".format(self.starlist) + batch_out += "trimfake={0}, ".format(self.trimfake) + + if self.stf_debug: + batch_out += "/debug, " + + batch_out += "fixPsf=1, " + batch_out += "backboxFWHM=25, " + batch_out += "flat=1, " + batch_out += "subtract=1" + + # Support for arbitrary starfinder flags. + batch_out += self.stfFlags + + batch_out += "\n" + batch_out += "exit\n" + + # Write out the current batch file's contents + with open(batch_file, 'w') as out_file: + out_file.write(batch_out) + + batch_files.append(batch_file) + batch_file_logs.append(batch_file_log) + + if self.stf_parallelize: + stf_pool = Pool(processes=4) + + stf_pool.map(idl_process_run, batch_files) + else: + for batch_file in batch_files: + idl_process_run(batch_file) - _batch.write("\n") - _batch.write("exit\n") - _batch.close() elif self.type == 'speckle': fileIDLbatch = 'idlbatch_combo' fileIDLlog = fileIDLbatch + '.log' @@ -311,11 +392,11 @@ def starfinderCombo(self, oldPsf=False): _batch.write("\n") _batch.write("exit\n") _batch.close() - - cmd = 'idl < ' + fileIDLbatch + ' >& ' + fileIDLlog - #os.system(cmd) - subp = subprocess.Popen(cmd, shell=True, executable="/bin/tcsh") - tmp = subp.communicate() + + cmd = 'idl < ' + fileIDLbatch + ' >& ' + fileIDLlog + #os.system(cmd) + subp = subprocess.Popen(cmd, shell=True, executable="/bin/tcsh") + tmp = subp.communicate() # Write data_sources file data_sources_file = open(self.dirComboStf + '/data_sources.txt', 'w') From 49b83e54ee417aa9f7b49af5f0ec666b08a6f530 Mon Sep 17 00:00:00 2001 From: Abhimat Local Date: Thu, 28 Sep 2023 17:00:57 -0700 Subject: [PATCH 2/6] Make stf parallelization the default --- kai/reduce/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kai/reduce/analysis.py b/kai/reduce/analysis.py index 5ff0946..4e954fe 100644 --- a/kai/reduce/analysis.py +++ b/kai/reduce/analysis.py @@ -47,7 +47,7 @@ def __init__(self, epoch, rootDir='/g/lu/data/orion/', filt='kp', epochDirSuffix=None, imgSuffix=None, stfDir=None, useDistorted=False, cleanList='c.lis', airopa_mode='single', stf_version=None, stf_debug=False, - stf_parallelize=False, + stf_parallelize=True, instrument=instruments.default_inst): """ Set up Analysis object. From dd64aef530ae4f156f8903341fdd3779230648d1 Mon Sep 17 00:00:00 2001 From: Abhimat Local Date: Thu, 28 Sep 2023 17:09:21 -0700 Subject: [PATCH 3/6] Combine all stf log files in the end for parallel runs --- kai/reduce/analysis.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kai/reduce/analysis.py b/kai/reduce/analysis.py index 4e954fe..1f01957 100644 --- a/kai/reduce/analysis.py +++ b/kai/reduce/analysis.py @@ -366,6 +366,14 @@ def starfinderCombo(self, oldPsf=False): for batch_file in batch_files: idl_process_run(batch_file) + # Combine all log files into one combo log file + # (to preserve backward compatibility for code that reads log files) + fileIDLlog = 'idlbatch_combo_' + self.filt + '.log' + with open(fileIDLlog, 'wb') as out_file: + for log_file in batch_file_logs: + with open(log_file, 'rb') as in_file: + out_file.write(in_file.read()) + elif self.type == 'speckle': fileIDLbatch = 'idlbatch_combo' fileIDLlog = fileIDLbatch + '.log' From 370243489e6c2caf2b96aaa475222fa37a8b00d9 Mon Sep 17 00:00:00 2001 From: zohaggard Date: Thu, 5 Oct 2023 11:50:47 -0700 Subject: [PATCH 4/6] make idl call work with multiple stf iterations in PSF creation --- kai/reduce/analysis.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/kai/reduce/analysis.py b/kai/reduce/analysis.py index 1f01957..0c9b2b7 100644 --- a/kai/reduce/analysis.py +++ b/kai/reduce/analysis.py @@ -86,9 +86,9 @@ def __init__(self, epoch, rootDir='/g/lu/data/orion/', filt='kp', """ # Setup default parameters self.type = 'ao' - self.corrMain = 0.8 - self.corrSub = 0.6 - self.corrClean = 0.7 + self.corrMain = [0.8] + self.corrSub = [0.6] + self.corrClean = [0.7] # airopa mode can be "legacy", "single", or "variable" self.airopa_mode = airopa_mode self.trimfake = 1 @@ -311,9 +311,9 @@ def starfinderCombo(self, oldPsf=False): batch_out += "'{0}', ".format(combo_file_path) if cur_combo == 'main': - batch_out += "{0:3.1f}, ".format(self.corrMain) + batch_out += "{0}, ".format(self.corrMain) else: - batch_out += "{0:3.1f}, ".format(self.corrSub) + batch_out += "{0}, ".format(self.corrSub) batch_out += "ttStar='TTstar', gsStar='', " @@ -346,7 +346,7 @@ def starfinderCombo(self, oldPsf=False): batch_out += "subtract=1" # Support for arbitrary starfinder flags. - batch_out += self.stfFlags + batch_out += ", {}".format(self.stfFlags) batch_out += "\n" batch_out += "exit\n" @@ -382,8 +382,8 @@ def starfinderCombo(self, oldPsf=False): _batch = open(fileIDLbatch, 'w') _batch.write("find_new_speck, ") _batch.write("'" + self.epoch + "', ") - _batch.write("corr_main=%3.1f, " % self.corrMain) - _batch.write("corr_subs=%3.1f, " % self.corrSub) + _batch.write("corr_main={0}, " % self.corrMain) + _batch.write("corr_subs={0}, " % self.corrSub) _batch.write("starlist='" + self.starlist + "', ") if self.airopa_mode == 'legacy': _batch.write("/legacy, ") @@ -653,14 +653,14 @@ def calibrateCombo(self): for ss in range(self.numSubMaps): if self.type == 'speckle': fileSub = 'm%s_%d_%3.1f_stf.lis' % \ - (self.epoch, ss+1, self.corrSub) + (self.epoch, ss+1, self.corrSub[0]) else: if self.deblend == 1: fileSub = 'm%s%s_%s_%d_%3.1fd_stf.lis' % \ - (self.epoch, self.imgSuffix, self.filt, ss+1, self.corrSub) + (self.epoch, self.imgSuffix, self.filt, ss+1, self.corrSub[0]) else: fileSub = 'm%s%s_%s_%d_%3.1f_stf.lis' % \ - (self.epoch, self.imgSuffix, self.filt, ss+1, self.corrSub) + (self.epoch, self.imgSuffix, self.filt, ss+1, self.corrSub[0]) print(cmd + fileSub) @@ -744,8 +744,8 @@ def alignCombo(self): os.chdir(self.dirComboAln) # Put the files in to the align*.list file - alnList1 = 'align%s%s_%3.1f.list' % (self.imgSuffix, file_ext, self.corrMain) - alnList2 = 'align%s%s_%3.1f_named.list' % (self.imgSuffix, file_ext, self.corrMain) + alnList1 = 'align%s%s_%3.1f.list' % (self.imgSuffix, file_ext, self.corrMain[0]) + alnList2 = 'align%s%s_%3.1f_named.list' % (self.imgSuffix, file_ext, self.corrMain[0]) _list = open(alnList1, 'w') @@ -758,10 +758,10 @@ def alignCombo(self): for ss in range(self.numSubMaps): if self.deblend == 1: _list.write('../m%s%s%s_%d_%3.1fd_stf_cal.lis %d\n' % - (self.epoch, self.imgSuffix, file_ext, ss+1, self.corrSub, alignType)) + (self.epoch, self.imgSuffix, file_ext, ss+1, self.corrSub[0], alignType)) else: _list.write('../m%s%s%s_%d_%3.1f_stf_cal.lis %d\n' % - (self.epoch, self.imgSuffix, file_ext, ss+1, self.corrSub, alignType)) + (self.epoch, self.imgSuffix, file_ext, ss+1, self.corrSub[0], alignType)) _list.close() From 6c0f627f369fc8f69bdf7fea0b2e0477b826d1f9 Mon Sep 17 00:00:00 2001 From: zohaggard Date: Thu, 5 Oct 2023 17:10:21 -0700 Subject: [PATCH 5/6] updates to correlation to work with mult. iters --- kai/reduce/analysis.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/kai/reduce/analysis.py b/kai/reduce/analysis.py index 0c9b2b7..d978e4b 100644 --- a/kai/reduce/analysis.py +++ b/kai/reduce/analysis.py @@ -635,10 +635,10 @@ def calibrateCombo(self): else: if self.deblend == 1: fileMain = 'mag%s%s_%s_%3.1fd_stf.lis' % \ - (self.epoch, self.imgSuffix, self.filt, self.corrMain) + (self.epoch, self.imgSuffix, self.filt, self.corrMain[0]) else: fileMain = 'mag%s%s_%s_%3.1f_stf.lis' % \ - (self.epoch, self.imgSuffix, self.filt, self.corrMain) + (self.epoch, self.imgSuffix, self.filt, self.corrMain[0]) print(cmd + fileMain) # Now call from within python... don't bother with command line anymore. @@ -751,10 +751,10 @@ def alignCombo(self): _list = open(alnList1, 'w') if self.deblend == 1: _list.write('../mag%s%s%s_%3.1fd_stf_cal.lis %d ref\n' % - (self.epoch, self.imgSuffix, file_ext, self.corrMain, alignType)) + (self.epoch, self.imgSuffix, file_ext, self.corrMain[0], alignType)) else: _list.write('../mag%s%s%s_%3.1f_stf_cal.lis %d ref\n' % - (self.epoch, self.imgSuffix, file_ext, self.corrMain, alignType)) + (self.epoch, self.imgSuffix, file_ext, self.corrMain[0], alignType)) for ss in range(self.numSubMaps): if self.deblend == 1: _list.write('../m%s%s%s_%d_%3.1fd_stf_cal.lis %d\n' % @@ -770,7 +770,7 @@ def alignCombo(self): # Make an unlabeled version cmd = 'java -Xmx1024m align %s ' % (self.alignFlags) - cmd += '-r align%s%s_%3.1f ' % (self.imgSuffix, file_ext, self.corrMain) + cmd += '-r align%s%s_%3.1f ' % (self.imgSuffix, file_ext, self.corrMain[0]) cmd += alnList1 print(cmd) #os.system(cmd) @@ -783,7 +783,7 @@ def alignCombo(self): cmd += '-accel_file %s ' % self.labellist if (self.orbitlist != None) and (self.orbitlist != ''): cmd += '-o %s ' % self.orbitlist - cmd += '-r align%s%s_%3.1f_named ' % (self.imgSuffix, file_ext, self.corrMain) + cmd += '-r align%s%s_%3.1f_named ' % (self.imgSuffix, file_ext, self.corrMain[0]) cmd += alnList2 print(cmd) @@ -792,33 +792,33 @@ def alignCombo(self): align_options = 'align%s%s_%3.1f %d -e' % \ - (self.imgSuffix, file_ext, self.corrMain, self.minSubMaps) + (self.imgSuffix, file_ext, self.corrMain[0], self.minSubMaps) align_rms.run(align_options.split()) align_options = 'align%s%s_%3.1f_named %d -e' % \ - (self.imgSuffix, file_ext, self.corrMain, self.minSubMaps) + (self.imgSuffix, file_ext, self.corrMain[0], self.minSubMaps) align_rms.run(align_options.split()) # Move the resulting files to their final resting place os.rename('align%s%s_%3.1f_rms.lis' % - (self.imgSuffix, file_ext, self.corrMain), + (self.imgSuffix, file_ext, self.corrMain[0]), '../mag%s%s%s_rms.lis' % (self.epoch, self.imgSuffix, file_ext)) os.rename('align%s%s_%3.1f_named_rms.lis' % - (self.imgSuffix, file_ext, self.corrMain), + (self.imgSuffix, file_ext, self.corrMain[0]), '../mag%s%s%s_rms_named.lis' % (self.epoch, self.imgSuffix, file_ext)) # Copy over the label.dat and orbit.dat file that was used. shutil.copyfile(self.labellist, 'align%s%s_%3.1f_named_label_list.txt' % - (self.imgSuffix, file_ext, self.corrMain)) + (self.imgSuffix, file_ext, self.corrMain[0])) if (self.orbitlist != None) and (self.orbitlist != ''): shutil.copyfile(self.orbitlist, 'align%s%s_%3.1f_named_orbit_list.txt' % - (self.imgSuffix, file_ext, self.corrMain)) + (self.imgSuffix, file_ext, self.corrMain[0])) # Now plot up the results plotSuffix = self.imgSuffix + file_ext From 7782bf8af38490785b6a9e1c5261cb91c586b072 Mon Sep 17 00:00:00 2001 From: Sean Granados <95511189+Upsilonlabsean@users.noreply.github.com> Date: Mon, 13 Nov 2023 20:14:25 -0800 Subject: [PATCH 6/6] added if statement for self.stfFlags to fix small bug with trailing comma --- kai/reduce/analysis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kai/reduce/analysis.py b/kai/reduce/analysis.py index d978e4b..01b2845 100644 --- a/kai/reduce/analysis.py +++ b/kai/reduce/analysis.py @@ -346,7 +346,8 @@ def starfinderCombo(self, oldPsf=False): batch_out += "subtract=1" # Support for arbitrary starfinder flags. - batch_out += ", {}".format(self.stfFlags) + if self.stfFlags != '': + batch_out += ", {0}".format(self.stfFlags) batch_out += "\n" batch_out += "exit\n"