diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index 29b2417..1804679 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -7,12 +7,17 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## +""" +This class acts as an entry-point into the psana-based utility of the library. +(should be used in high-level scripts as follows: 'from calibrationSuite.basicSuiteScript import BasicSuiteScript') +It contains utility functions, analysis functions, and some getters and setters, that can be used with both 'psana1Base' and 'psana2Base'. +""" + import logging import os import numpy as np - -from calibrationSuite.detectorInfo import DetectorInfo +import psana logger = logging.getLogger(__name__) @@ -24,230 +29,105 @@ from calibrationSuite.psana2Base import PsanaBase -def sortArrayByList(a, data): - return [x for _, x in sorted(zip(a, data), key=lambda pair: pair[0])] - - class BasicSuiteScript(PsanaBase): def __init__(self, analysisType="scan"): super().__init__() + print("in BasicSuiteScript, inheriting from PsanaBase, type is psana%d" % (self.psanaType)) logger.info("in BasicSuiteScript, inheriting from PsanaBase, type is psana%d" % (self.psanaType)) + self.className = self.__class__.__name__ - self.gainModes = {"FH": 0, "FM": 1, "FL": 2, "AHL-H": 3, "AML-M": 4, "AHL-L": 5, "AML-L": 6} - self.ePix10k_cameraTypes = {1: "Epix10ka", 4: "Epix10kaQuad", 16: "Epix10ka2M"} - self.camera = 0 + #### Start of common getter functions #### - self.outputDir = "/%s/" % (analysisType) - logger.info("output dir: " + self.outputDir) - print("output dir: " + self.outputDir) + def get_evrs(self): + if self.config is None: + self.get_config() - try: - self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"], self.experimentHash["detectorSubtype"]) - except: - self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"]) - self.className = self.__class__.__name__ + self.evrs = [] + for key in list(self.config.keys()): + if key.type() == psana.EvrData.ConfigV7: + self.evrs.append(key.src()) - try: - self.location = self.experimentHash["location"] - except Exception: - pass - try: - self.exp = self.experimentHash["exp"] - except Exception: - pass - self.ROIfileNames = None - try: - ##if True: - self.ROIfileNames = self.experimentHash["ROIs"] - self.ROIs = [] - for f in self.ROIfileNames: - self.ROIs.append(np.load(f)) - try: ## dumb code for compatibility or expectation - self.ROI = self.ROIs[0] - except Exception: - pass - ##if False: - except Exception: - if self.ROIfileNames is not None: - print("had trouble finding", self.ROIfileNames) - for currName in self.ROIfileNames: - logger.exception("had trouble finding" + currName) - self.ROI = None - self.ROIs = [] - try: - self.singlePixels = self.experimentHash["singlePixels"] - except Exception: - self.singlePixels = None - try: - self.regionSlice = self.experimentHash["regionSlice"] - except Exception: - self.regionSlice = None - if self.regionSlice is not None: - ## n.b. assumes 3d slice now - self.sliceCoordinates = [ - [self.regionSlice[1].start, self.regionSlice[1].stop], - [self.regionSlice[2].start, self.regionSlice[2].stop], - ] - sc = self.sliceCoordinates - self.sliceEdges = [sc[0][1] - sc[0][0], sc[1][1] - sc[1][0]] - - ## handle 1d rixs ccd data - if self.detectorInfo.dimension == 2: - self.regionSlice = self.regionSlice[0], self.regionSlice[2] - print("remapping regionSlice to handle 1d case") - - try: - self.fluxSource = self.experimentHash["fluxSource"] + def get_config(self): + self.config = self.ds.env().configStore() + + def getFivePedestalRunInfo(self): + ## could do load_txt but would require full path so + if self.det is None: + self.setupPsana() + + evt = self.getEvt(self.fivePedestalRun) + self.fpGains = self.det.gain(evt) + self.fpPedestals = self.det.pedestals(evt) + self.fpStatus = self.det.status(evt) ## does this work? + self.fpRMS = self.det.rms(evt) ## does this work? + + def getEvtFromRunsTooSmartForMyOwnGood(self): + for r in self.runRange: + self.run = r + self.ds = self.get_ds() try: - self.fluxChannels = self.experimentHash["fluxChannels"] + evt = next(self.ds.events()) + yield evt except Exception: - self.fluxChannels = range(8, 16) ## wave8 + continue + + def getEvtFromRuns(self): + try: ## can't get yield to work + evt = next(self.ds.events()) + return evt + except StopIteration: + i = self.runRange.index(self.run) try: - self.fluxSign = self.experimentHash["fluxSign"] + self.run = self.runRange[i + 1] + print("switching to run %d" % (self.run)) + logger.info("switching to run %d" % (self.run)) + self.ds = self.get_ds(self.run) except Exception: - self.fluxSign = 1 - except Exception: - self.fluxSource = None - - try: - self.ignoreEventCodeCheck = self.experimentHash["ignoreEventCodeCheck"] - self.fakeBeamCode = True ## just in case - except Exception: - self.ignoreEventCodeCheck = False - self.fakeBeamCode = False - - self.special = self.args.special - ## for non-120 Hz running - self.nRunCodeEvents = 0 - self.nDaqCodeEvents = 0 - self.nBeamCodeEvents = 0 - self.runCode = 280 - self.daqCode = 281 - self.beamCode = 283 ## per Matt - ##self.beamCode = 281 ## don't see 283... - if not self.fakeBeamCode: ## defined in ignoreEventCodeCheck - if self.special is not None: - self.fakeBeamCode = "fakeBeamCode" in self.special - - print("ignoring event code check, faking beam code:", self.ignoreEventCodeCheck, self.fakeBeamCode) - - ##mymodule = importlib.import_module(full_module_name) - - ## for standalone analysis - self.file = None - if self.args.files is not None: - self.file = self.args.files - self.label = "" - if self.args.label is not None: - self.label = self.args.label - - ## analyzing xtcs - if self.args.run is not None: - self.run = self.args.run - if self.args.camera is not None: - self.camera = self.args.camera - if self.args.exp is not None: - self.exp = self.args.exp - if self.args.location is not None: - self.location = self.args.location - if self.args.maxNevents is not None: - self.maxNevents = self.args.maxNevents - if self.args.skipNevents is not None: - self.skipNevents = self.args.skipNevents - if self.args.path is not None: - self.outputDir = self.args.path - # if set, output folders will be relative to OUTPUT_ROOT - # if not, they will be relative to the current script file - self.outputDir = os.getenv("OUTPUT_ROOT", ".") + self.outputDir - # check if outputDir exists, if does not create it and tell user - if not os.path.exists(self.outputDir): - print("could not find output dir: " + self.outputDir) - logger.info("could not find output dir: " + self.outputDir) - print("please create this dir, exiting...") - logger.info("please create this dir, exiting...") - exit(1) - # the following doesnt work with mpi parallelism (other thread could make dir b4 curr thread) - # print("so creating dir: " + self.outputDir) - # logger.info("creating dir: " + self.outputDir) - # os.makedirs(self.outputDir) - # give dir read, write, execute permissions - # os.chmod(self.outputDir, 0o777) - self.detObj = self.args.detObj - if self.args.threshold is not None: - self.threshold = eval(self.args.threshold) - else: - self.threshold = None - if self.args.seedCut is not None: - self.seedCut = eval(self.args.seedCut) - else: - self.seedCut = None - if self.args.fluxCutMin is not None: - self.fluxCutMin = self.args.fluxCutMin - if self.args.fluxCutMax is not None: - self.fluxCutMax = self.args.fluxCutMax - try: - self.runRange = eval(self.args.runRange) ## in case needed - except Exception: - self.runRange = None - - self.fivePedestalRun = self.args.fivePedestalRun ## in case needed - self.fakePedestal = None - self.fakePedestalFile = self.args.fakePedestalFile ## in case needed - if self.fakePedestalFile is not None: - self.fakePedestal = np.load(self.fakePedestalFile) ##cast to uint32??? - - self.g0PedFile = self.args.g0PedFile - if self.g0PedFile is not None: - ##self.g0Ped = np.load(self.g0PedFile) - self.g0Ped = np.array([np.load(self.g0PedFile)]) ##temp hack - print(self.g0Ped.shape) - - self.g1PedFile = self.args.g0PedFile - if self.g1PedFile is not None: - ##self.g1Ped = np.load(self.g1PedFile) - self.g1Ped = np.array([np.load(self.g1PedFile)]) ##temp hack - - self.g0GainFile = self.args.g0GainFile - if self.g0GainFile is not None: - self.g0Gain = np.load(self.g0GainFile) - - self.g1GainFile = self.args.g1GainFile - if self.g1GainFile is not None: - self.g1Gain = np.load(self.g1GainFile) - - self.offsetFile = self.args.offsetFile - if self.offsetFile is not None: - self.offset = np.load(self.offsetFile) - - if self.args.detType == "": - ## assume epix10k for now - if self.args.nModules is not None: - self.detectorInfo.setNModules(self.args.nModules) - self.detType = self.detectorInfo.getCameraType() - else: - self.detType = self.args.detType - - try: - self.analyzedModules = self.experimentHash["analyzedModules"] - except Exception: - self.analyzedModules = range(self.detectorInfo.nModules) - - self.g0cut = self.detectorInfo.g0cut - if self.g0cut is not None: - self.gainBitsMask = self.g0cut - 1 - else: - self.gainBitsMask = 0xFFFF ## might be dumb. for non-autoranging - - self.negativeGain = self.detectorInfo.negativeGain ## could just use the detector info in places it's defined - - ## done with configuration - - self.ds = None - self.det = None ## do we need multiple dets in an array? or self.secondDet? - - ##self.setupPsana() - ##do this later or skip for -file + print("have run out of new runs") + logger.exception("have run out of new runs") + return None + ##print("get event from new run") + evt = next(self.ds.events()) + return evt + + def getRawData(self, evt, gainBitsMasked=True, negativeGain=False): + frames = self.plainGetRawData(evt) + if frames is None: + return None + if False and self.special: ## turned off for a tiny bit of speed + if "thirteenBits" in self.special: + frames = frames & 0xFFFE + ##print("13bits") + elif "twelveBits" in self.special: + frames = frames & 0xFFFC + ##print("12bits") + elif "elevenBits" in self.special: + frames = frames & 0xFFF8 + ##print("11bits") + elif "tenBits" in self.special: + frames = frames & 0xFFF0 + ##print("10bits") + if self.negativeGain or negativeGain: + zeroPixels = frames == 0 + maskedData = frames & self.gainBitsMask + gainData = frames - maskedData + frames = gainData + self.gainBitsMask - maskedData + frames[zeroPixels] = 0 + if gainBitsMasked: + return frames & self.gainBitsMask + return frames + + def addFakePhotons(self, frames, occupancy, E, width): + shape = frames.shape + occ = np.random.random(shape) + fakes = np.random.normal(E, width, shape) + fakes[occ > occupancy] = 0 + return frames + fakes, (fakes > 0).sum() + + #### End of common getter functions #### + + #### Start of common setter functions #### def setROI(self, roiFile=None, roi=None): """Call with both file name and roi to save roi to file and use, @@ -262,9 +142,50 @@ def setROI(self, roiFile=None, roi=None): np.save(roiFile, roi) self.ROI = roi + #### End of common setter functions #### + + #### Start of common utility functions #### + + def sortArrayByList(self, a, data): + return [x for _, x in sorted(zip(a, data), key=lambda pair: pair[0])] + def sliceToDetector(self, sliceRow, sliceCol): ## cp from AnalyzeH5: import? return sliceRow + self.sliceCoordinates[0][0], sliceCol + self.sliceCoordinates[1][0] + def getNswitchedPixels(self, data, region=None): + return ((data >= self.g0cut) * 1).sum() + + def dumpEventCodeStatistics(self): + print( + "have counted %d run triggers, %d DAQ triggers, %d beam events" + % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) + ) + logger.info( + "have counted %d run triggers, %d DAQ triggers, %d beam events" + % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) + ) + + def isBeamEvent(self, evt): + if self.ignoreEventCodeCheck: + return True + ec = self.getEventCodes(evt) + ##print(ec[280], ec[281], ec[282], ec[283], ec[284], ec[285] ) + if ec[self.runCode]: + self.nRunCodeEvents += 1 + if ec[self.daqCode]: + self.nDaqCodeEvents += 1 + if self.fakeBeamCode: + return True + if ec[self.beamCode]: + self.nBeamCodeEvents += 1 + return True + ## for FEE, ASC, ... + return self.fakeBeamCode ##False + + #### End of common utility functions #### + + #### Start of analysis/correction functions #### + def noCommonModeCorrection(self, frames): return frames @@ -331,78 +252,4 @@ def colCommonModeCorrection(self, frame, arbitraryCut=1000): rowOffset += self.detectorInfo.nRowsPerBank return frame - def isBeamEvent(self, evt): - if self.ignoreEventCodeCheck: - return True - ec = self.getEventCodes(evt) - ##print(ec[280], ec[281], ec[282], ec[283], ec[284], ec[285] ) - if ec[self.runCode]: - self.nRunCodeEvents += 1 - if ec[self.daqCode]: - self.nDaqCodeEvents += 1 - if self.fakeBeamCode: - return True - if ec[self.beamCode]: - self.nBeamCodeEvents += 1 - return True - ## for FEE, ASC, ... - return self.fakeBeamCode ##False - - def dumpEventCodeStatistics(self): - print( - "have counted %d run triggers, %d DAQ triggers, %d beam events" - % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) - ) - logger.info( - "have counted %d run triggers, %d DAQ triggers, %d beam events" - % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) - ) - - def getRawData(self, evt, gainBitsMasked=True, negativeGain=False): - frames = self.plainGetRawData(evt) - if frames is None: - return None - if False and self.special: ## turned off for a tiny bit of speed - if "thirteenBits" in self.special: - frames = frames & 0xFFFE - ##print("13bits") - elif "twelveBits" in self.special: - frames = frames & 0xFFFC - ##print("12bits") - elif "elevenBits" in self.special: - frames = frames & 0xFFF8 - ##print("11bits") - elif "tenBits" in self.special: - frames = frames & 0xFFF0 - ##print("10bits") - if self.negativeGain or negativeGain: - zeroPixels = frames == 0 - maskedData = frames & self.gainBitsMask - gainData = frames - maskedData - frames = gainData + self.gainBitsMask - maskedData - frames[zeroPixels] = 0 - if gainBitsMasked: - return frames & self.gainBitsMask - return frames - - def addFakePhotons(self, frames, occupancy, E, width): - shape = frames.shape - occ = np.random.random(shape) - fakes = np.random.normal(E, width, shape) - fakes[occ > occupancy] = 0 - return frames + fakes, (fakes > 0).sum() - - def getNswitchedPixels(self, data, region=None): - return ((data >= self.g0cut) * 1).sum() - - -""" -if __name__ == "__main__": - bSS = BasicSuiteScript() - print("have built a BasicSuiteScript") - logger.info("have built a BasicSuiteScript") - bSS.setupPsana() - evt = bSS.getEvt() - print(dir(evt)) - logger.info(dir(evt)) -""" + #### End of analysis/correction functions #### diff --git a/calibrationSuite/detectorInfo.py b/calibrationSuite/detectorInfo.py index f48edd5..2347543 100644 --- a/calibrationSuite/detectorInfo.py +++ b/calibrationSuite/detectorInfo.py @@ -8,7 +8,7 @@ ## the terms contained in the LICENSE.txt file. ############################################################################## class DetectorInfo: - def __init__(self, detType, detSubtype=None): + def __init__(self, detType, detSubtype="1d"): # declare all detector-specific info vars here in case any setup_X functions don't, # and use -1 so caller knows things are not setup (non-0 to avoid error on divide. self.nModules = -1 @@ -88,7 +88,7 @@ def setup_epixM(self, version=0): self.seedCut = 2 self.neighborCut = 0.25 ## ditto - def setup_rixsCCD(self, mode="1d", version=0): + def setup_rixsCCD(self, mode, version=0): print("rixsCCD mode:", mode) self.nTestPixelsPerBank = 36 self.nBanks = 16 diff --git a/calibrationSuite/psana1Base.py b/calibrationSuite/psana1Base.py index d010b2e..aedc211 100755 --- a/calibrationSuite/psana1Base.py +++ b/calibrationSuite/psana1Base.py @@ -7,69 +7,34 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## -# from psana import * -import importlib +""" +This class contains a setup function and some utility functions that work only for psana1 based analysis. +To make the library use this class execute 'export foo=psana1'. +""" + import logging -import os import sys import psana -from calibrationSuite.argumentParser import ArgumentParser +from calibrationSuite.psanaCommon import PsanaCommon logger = logging.getLogger(__name__) -class PsanaBase(object): +class PsanaBase(PsanaCommon): def __init__(self, analysisType="scan"): + super().__init__() + commandUsed = sys.executable + " " + " ".join(sys.argv) logger.info("Ran with cmd: " + commandUsed) self.psanaType = 1 print("in psana1Base") logger.info("in psana1Base") - self.gainModes = {"FH": 0, "FM": 1, "FL": 2, "AHL-H": 3, "AML-M": 4, "AHL-L": 5, "AML-L": 6} - self.ePix10k_cameraTypes = {1: "Epix10ka", 4: "Epix10kaQuad", 16: "Epix10ka2M"} self.g0cut = 1 << 14 self.gainBitsMask = self.g0cut - 1 - self.args = ArgumentParser().parse_args() - logger.info("parsed cmdline args: " + str(self.args)) - - # if the SUITE_CONFIG env var is set use that, otherwise if the cmd line arg is set use that - # if neither are set, use the default 'suiteConfig.py' file - defaultConfigFileName = "suiteConfig.py" - secondaryConfigFileName = defaultConfigFileName if self.args.configFile is None else self.args.configFile - # secondaryConfigFileName is returned if env var not set - configFileName = os.environ.get("SUITE_CONFIG", secondaryConfigFileName) - config = self.importConfigFile(configFileName) - if config is None: - print("\ncould not find or read config file: " + configFileName) - print("please set SUITE_CONFIG env-var or use the '-cf' cmd-line arg to specify a valid config file") - print("exiting...") - sys.exit(1) - self.experimentHash = config.experimentHash - knownTypes = ["epixhr", "epixM", "rixsCCD"] - if self.experimentHash["detectorType"] not in knownTypes: - print("type %s not in known types" % (self.experimentHash["detectorType"]), knownTypes) - return -1 - - ## self.setupPsana() - - def importConfigFile(self, file_path): - if not os.path.exists(file_path): - print(f"The file '{file_path}' does not exist") - return None - spec = importlib.util.spec_from_file_location("config", file_path) - config_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(config_module) - return config_module - - def get_ds(self, run=None): - if run is None: - run = self.run - return psana.DataSource("exp=%s:run=%d:smd" % (self.exp, run)) - def setupPsana(self): logger.info("have built basic script class, exp %s run %d" % (self.exp, self.run)) @@ -91,16 +56,10 @@ def setupPsana(self): except Exception: self.controlData = None - def getFivePedestalRunInfo(self): - ## could do load_txt but would require full path so - if self.det is None: - self.setupPsana() - - evt = self.getEvt(self.fivePedestalRun) - self.fpGains = self.det.gain(evt) - self.fpPedestals = self.det.pedestals(evt) - self.fpStatus = self.det.status(evt) ## does this work? - self.fpRMS = self.det.rms(evt) ## does this work? + def get_ds(self, run=None): + if run is None: + run = self.run + return psana.DataSource("exp=%s:run=%d:smd" % (self.exp, run)) def getEvt(self, run=None): oldDs = self.ds @@ -114,35 +73,6 @@ def getEvt(self, run=None): self.ds = oldDs return evt - def getEvtFromRunsTooSmartForMyOwnGood(self): - for r in self.runRange: - self.run = r - self.ds = self.get_ds() - try: - evt = next(self.ds.events()) - yield evt - except Exception: - continue - - def getEvtFromRuns(self): - try: ## can't get yield to work - evt = next(self.ds.events()) - return evt - except StopIteration: - i = self.runRange.index(self.run) - try: - self.run = self.runRange[i + 1] - print("switching to run %d" % (self.run)) - logger.info("switching to run %d" % (self.run)) - self.ds = self.get_ds(self.run) - except Exception: - print("have run out of new runs") - logger.exception("have run out of new runs") - return None - ##print("get event from new run") - evt = next(self.ds.events()) - return evt - def getFlux(self, evt): try: fluxes = self.wave8.get(evt).peakA() @@ -165,15 +95,6 @@ def getFlux(self, evt): return None return f - def get_evrs(self): - if self.config is None: - self.get_config() - - self.evrs = [] - for key in list(self.config.keys()): - if key.type() == psana.EvrData.ConfigV7: - self.evrs.append(key.src()) - def isKicked(self, evt): try: evr = evt.get(psana.EvrData.DataV4, self.evrs[0]) @@ -195,9 +116,6 @@ def isKicked(self, evt): pass return kicked - def get_config(self): - self.config = self.ds.env().configStore() - def getStepGen(self): return self.ds.steps() @@ -217,13 +135,3 @@ def getCalibData(self, evt): def getImage(self, evt, data=None): return self.raw.image(evt, data) - - -""" -if __name__ == "__main__": - bSS = BasicSuiteScript() - print("have built a BasicSuiteScript") - bSS.setupPsana() - evt = bSS.getEvt() - print(dir(evt)) -""" diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index 9900b93..664f963 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -7,11 +7,12 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## -# from psana import * -import importlib.util -import logging +""" +This class contains a setup function and some utility functions that work only for psana2 based analysis. +To make the library use this class execute 'export foo=psana2'. +""" -## for parallelism +import logging import os import sys @@ -20,7 +21,7 @@ ## standard from mpi4py import MPI -from calibrationSuite.argumentParser import ArgumentParser +from calibrationSuite.psanaCommon import PsanaCommon ##from PSCalib.NDArrIO import load_txt @@ -38,8 +39,10 @@ logger = logging.getLogger(__name__) -class PsanaBase(object): +class PsanaBase(PsanaCommon): def __init__(self, analysisType="scan"): + super().__init__() + commandUsed = sys.executable + " " + " ".join(sys.argv) logger.info("Ran with cmd: " + commandUsed) @@ -47,54 +50,15 @@ def __init__(self, analysisType="scan"): print("in psana2Base") logger.info("in psana2Base") - self.gainModes = {"FH": 0, "FM": 1, "FL": 2, "AHL-H": 3, "AML-M": 4, "AHL-L": 5, "AML-L": 6} - self.ePix10k_cameraTypes = {1: "Epix10ka", 4: "Epix10kaQuad", 16: "Epix10ka2M"} - ##self.g0cut = 1<<15 ## 2022 - self.allowed_timestamp_mismatch = 1000 - self.args = ArgumentParser().parse_args() - logger.info("parsed cmdline args: " + str(self.args)) - - # if the SUITE_CONFIG env var is set use that, otherwise if the cmd line arg is set use that - # if neither are set, use the default 'suiteConfig.py' file - defaultConfigFileName = "suiteConfig.py" - secondaryConfigFileName = defaultConfigFileName if self.args.configFile is None else self.args.configFile - # secondaryConfigFileName is returned if env var not set - configFileName = os.environ.get("SUITE_CONFIG", secondaryConfigFileName) - config = self.importConfigFile(configFileName) - if config is None: - print("\ncould not find or read config file: " + configFileName) - print("please set SUITE_CONFIG env-var or use the '-cf' cmd-line arg to specify a valid config file") - print("exiting...") - sys.exit(1) - self.experimentHash = config.experimentHash - knownTypes = ["epixhr", "epixM", "rixsCCD"] - if self.experimentHash["detectorType"] not in knownTypes: - print("type %s not in known types" % (self.experimentHash["detectorType"]), knownTypes) - return -1 - ##self.setupPsana() - - def importConfigFile(self, file_path): - if not os.path.exists(file_path): - print(f"The file '{file_path}' does not exist") - return None - spec = importlib.util.spec_from_file_location("config", file_path) - config_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(config_module) - return config_module - - def get_ds(self, run=None): - if run is None: - run = self.run - ##tmpDir = '/sdf/data/lcls/ds/rix/rixx1005922/scratch/xtc'## temp - ds = psana.DataSource( - exp=self.exp, run=run, intg_det=self.experimentHash["detectorType"], max_events=self.maxNevents - ) ##, dir=tmpDir) - return ds - def setupPsana(self): - ##print("have built basic script class, exp %s run %d" %(self.exp, self.run)) + ## fix hardcoding in the fullness of time + self.detEvts = 0 + self.flux = None + self.config = None + self.evrs = None + if self.runRange is None: self.ds = self.get_ds(self.run) else: @@ -105,11 +69,10 @@ def setupPsana(self): try: self.step_value = self.myrun.Detector("step_value") self.step_docstring = self.myrun.Detector("step_docstring") - ##print('foo', self.step_value, self.step_docstring) except Exception: self.step_value = self.step_docstring = None - ## self.det = Detector('%s.0:%s.%d' %(self.location, self.detType, self.camera), self.ds.env()) + ## self.det = Detector('%s.0:%s.%d' %(self.location, self.detType, self.camera), self.ds.env()) ## make this less dumb to accomodate epixM etc. ## use a dict etc. self.det = self.myrun.Detector(self.experimentHash["detectorType"]) @@ -127,37 +90,30 @@ def setupPsana(self): self.mfxDg1 = None print("No flux source found") ## if self.verbose? logger.exception("No flux source found") + try: self.mfxDg2 = self.myrun.Detector("MfxDg2BmMon") except Exception: self.mfxDg2 = None - ## fix hardcoding in the fullness of time - self.detEvts = 0 - self.flux = None - self.evrs = None try: self.wave8 = psana.Detector(self.fluxSource, self.ds.env()) except Exception: self.wave8 = None - self.config = None + try: self.controlData = psana.Detector("ControlData") except Exception: self.controlData = None - ## if self.mfxDg1 is None: - - def getFivePedestalRunInfo(self): - ## could do load_txt but would require full path so - if self.det is None: - self.setupPsana() - - evt = self.getEvt(self.fivePedestalRun) - self.fpGains = self.det.gain(evt) - self.fpPedestals = self.det.pedestals(evt) - self.fpStatus = self.det.status(evt) ## does this work? - self.fpRMS = self.det.rms(evt) ## does this work? + def get_ds(self, run=None): + if run is None: + run = self.run + ##tmpDir = '/sdf/data/lcls/ds/rix/rixx1005922/scratch/xtc' + ds = psana.DataSource( + exp=self.exp, run=run, intg_det=self.experimentHash["detectorType"], max_events=self.maxNevents + ) ##, dir=tmpDir) + return ds def getEvtOld(self, run=None): oldDs = self.ds @@ -201,35 +157,6 @@ def matchedDetEvt(self): else: continue - def getEvtFromRunsTooSmartForMyOwnGood(self): - for r in self.runRange: - self.run = r - self.ds = self.get_ds() - try: - evt = next(self.ds.events()) - yield evt - except Exception: - continue - - def getEvtFromRuns(self): - try: ## can't get yield to work - evt = next(self.ds.events()) - return evt - except StopIteration: - i = self.runRange.index(self.run) - try: - self.run = self.runRange[i + 1] - print("switching to run %d" % (self.run)) - logger.info("switching to run %d" % (self.run)) - self.ds = self.get_ds(self.run) - except Exception: - print("have run out of new runs") - logger.exception("have run out of new runs") - return None - ##print("get event from new run") - evt = next(self.ds.events()) - return evt - def getAllFluxes(self, evt): if evt is None: return None @@ -242,7 +169,7 @@ def _getFlux(self, evt): if self.mfxDg1 is None: return None - ## f = self.mfxDg1.raw.peakAmplitude(evt)[self.fluxChannels].mean()*self.fluxSign + ##f = self.mfxDg1.raw.peakAmplitude(evt)[self.fluxChannels].mean()*self.fluxSign try: f = self.mfxDg1.raw.peakAmplitude(evt)[self.fluxChannels].mean() * self.fluxSign ##print(f) @@ -265,15 +192,6 @@ def getFlux(self, evt): ##return 1 return self.flux - def get_evrs(self): - if self.config is None: - self.get_config() - - self.evrs = [] - for key in list(self.config.keys()): - if key.type() == psana.EvrData.ConfigV7: - self.evrs.append(key.src()) - def getEventCodes(self, evt): return self.timing.raw.eventcodes(evt) @@ -285,9 +203,6 @@ def isKicked(self, evt): ##print(allcodes) return allcodes[self.desiredCodes["120Hz"]] - def get_config(self): - self.config = self.ds.env().configStore() - def getStepGen(self): return self.myrun.steps() @@ -338,13 +253,3 @@ def getPingPongParity(self, frameRegion): delta = evensEvenRowsOddsOddRows.mean() - oddsEvenRowsEvensOddRows.mean() ##print("delta:", delta) return delta > 0 - - -""" -if __name__ == "__main__": - bSS = BasicSuiteScript() - print("have built a BasicSuiteScript") - bSS.setupPsana() - evt = bSS.getEvt() - print(dir(evt)) -""" diff --git a/calibrationSuite/psanaCommon.py b/calibrationSuite/psanaCommon.py new file mode 100755 index 0000000..0c62932 --- /dev/null +++ b/calibrationSuite/psanaCommon.py @@ -0,0 +1,343 @@ +############################################################################## +## This file is part of 'SLAC Beamtime Calibration Suite'. +## It is subject to the license terms in the LICENSE.txt file found in the +## top-level directory of this distribution and at: +## https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +## No part of 'SLAC Beamtime Calibration Suite', including this file, +## may be copied, modified, propagated, or distributed except according to +## the terms contained in the LICENSE.txt file. +############################################################################## +""" +This class has setup-code and variables common for both psana bases. +(psana1Base and psana2Base must inherit from this class! ('PsanaCommon')) +""" + +import importlib.util +import logging + +## for parallelism +import os +import sys + +import numpy as np + +from calibrationSuite.argumentParser import ArgumentParser +from calibrationSuite.detectorInfo import DetectorInfo + +logger = logging.getLogger(__name__) + + +class PsanaCommon(object): + def __init__(self, analysisType="scan"): + print("in psanaCommon") + logger.info("in psanaCommon") + + self.args = ArgumentParser().parse_args() + logger.info("parsed cmdline args: " + str(self.args)) + + self.gainModes = {"FH": 0, "FM": 1, "FL": 2, "AHL-H": 3, "AML-M": 4, "AHL-L": 5, "AML-L": 6} + self.ePix10k_cameraTypes = {1: "Epix10ka", 4: "Epix10kaQuad", 16: "Epix10ka2M"} + + self.ds = None + self.det = None ## do we need multiple dets in an array? or self.secondDet? + + ## for non-120 Hz running + self.nRunCodeEvents = 0 + self.nDaqCodeEvents = 0 + self.nBeamCodeEvents = 0 + self.runCode = 280 + self.daqCode = 281 + self.beamCode = 283 ## per Matt + ##self.beamCode = 281 ## don't see 283... + + # Variable-setup calls + self.loadExperimentHashFromConfig() + self.setupFromExperimentHash() + self.setupFromCmdlineArgs() + self.setupOutputDirString(analysisType) + + #### Start of setup related functions #### + + def loadExperimentHashFromConfig(self): + """ + Loads the experiment config file (ex: beamtime-calibration-suite/config_files/epixMSuiteConfig.py) + The config file is determined by checking the SUITE_CONFIG environment variable, + then a command-line argument, and finally a default file 'suiteConfig.py' (in this order of precedence). + If file cannot be found or read, exit the program. + """ + + # if the SUITE_CONFIG env var is set use that, otherwise if the cmd line arg is set use that + # if neither are set, use the default 'suiteConfig.py' file + defaultConfigFileName = "suiteConfig.py" + secondaryConfigFileName = defaultConfigFileName if self.args.configFile is None else self.args.configFile + # secondaryConfigFileName is returned if env var not set + configFileName = os.environ.get("SUITE_CONFIG", secondaryConfigFileName) + config = self.importConfigFile(configFileName) + if config is None: + print("\ncould not find or read config file: " + configFileName) + print("please set SUITE_CONFIG env-var or use the '-cf' cmd-line arg to specify a valid config file") + print("exiting...") + logger.error("\ncould not find or read config file: " + configFileName) + logger.error("please set SUITE_CONFIG env-var or use the '-cf' cmd-line arg to specify a valid config file") + logger.error("exiting...") + sys.exit(1) + self.experimentHash = config.experimentHash + + def importConfigFile(self, file_path): + """ + Imports config file as a Python module. See loadExperimentHashFromConfig() for how the config file + is specified by the user. + """ + if not os.path.exists(file_path): + print("The file " + file_path + " does not exist") + logger.error("The file " + file_path + " does not exist") + return None + spec = importlib.util.spec_from_file_location("config", file_path) + config_module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(config_module) + except Exception as e: + print("Error executing config-file code: " + str(e)) + logger.info("Error executing config-file code: " + str(e)) + return None + return config_module + + def setupFromExperimentHash(self): + """ + Sets up configurations and parameters based on user-specified config file. + Initializes necessary attributes and handles exceptions on specific evaluations. + """ + + try: + self.detectorInfo = DetectorInfo( + self.experimentHash["detectorType"], self.experimentHash["detectorSubtype"] + ) + except Exception: + self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"]) + + self.exp = self.experimentHash.get("exp", None) + + self.setupROIFiles() + + self.singlePixels = self.experimentHash.get("singlePixels", None) + + self.regionSlice = self.experimentHash.get("regionSlice", None) + if self.regionSlice is not None: + ## n.b. assumes 3d slice now + self.sliceCoordinates = [ + [self.regionSlice[1].start, self.regionSlice[1].stop], + [self.regionSlice[2].start, self.regionSlice[2].stop], + ] + sc = self.sliceCoordinates + self.sliceEdges = [sc[0][1] - sc[0][0], sc[1][1] - sc[1][0]] + + ## handle 1d rixs ccd data + if self.detectorInfo.dimension == 2: + self.regionSlice = self.regionSlice[0], self.regionSlice[2] + print("remapping regionSlice to handle 1d case") + logger.info("remapping regionSlice to handle 1d case") + + self.fluxSource = self.experimentHash.get("fluxSource", None) + self.fluxChannels = self.experimentHash.get("fluxChannels", range(8, 16)) ## wave8 + self.fluxSign = self.experimentHash.get("fluxSign", 1) + + self.ignoreEventCodeCheck = self.experimentHash.get("ignoreEventCodeCheck", None) + self.fakeBeamCode = True if self.ignoreEventCodeCheck is not None else False + + def setupROIFiles(self): + """ + Sets up the ROI (Region of Interest) files. + Loads ROI data from the file names specified in the config file. + If any errors occur during the process, logs and prints appropriate error messages. + """ + + self.ROIfileNames = None + try: + self.ROIfileNames = self.experimentHash.get("ROIs", None) + self.ROIs = [] + for f in self.ROIfileNames: + self.ROIs.append(np.load(f)) + self.ROI = self.ROIs[0] if len(self.ROIs) >= 1 else None + except Exception: + if self.ROIfileNames is not None: + print("had trouble finding" + str(self.ROIfileNames)) + logger.error("had trouble finding" + str(self.ROIfileNames)) + for currName in self.ROIfileNames: + print("had trouble finding" + currName) + logger.exception("had trouble finding" + currName) + + def setupFromCmdlineArgs(self): + """ + Sets up configurations and parameters based on user-entered command-line arguments. + Initializes necessary attributes and handles exceptions on specific evaluations. + """ + + self.special = self.args.special + + if not self.fakeBeamCode: ## defined in ignoreEventCodeCheck + if self.special is not None: + self.fakeBeamCode = "fakeBeamCode" in self.special + + print( + "ignoring event code check, faking beam code:" + + str(self.ignoreEventCodeCheck) + + " " + + str(self.fakeBeamCode) + ) + logger.info( + "ignoring event code check, faking beam code:" + + str(self.ignoreEventCodeCheck) + + " " + + str(self.fakeBeamCode) + ) + + ## for standalone analysis + self.file = self.args.files + self.label = "" if self.args.label is None else self.args.label + + ## analyzing xtcs + self.run = self.args.run + + self.camera = 0 if self.args.camera is None else self.args.camera + + # this is set in the config-file, but take the cmd-line value instead if it is set + if self.args.exp is not None: + self.exp = self.args.exp + + self.location = self.experimentHash.get("location", None) + if self.args.location is not None: + self.location = self.args.location + + self.maxNevents = self.args.maxNevents + self.skipNevents = self.args.skipNevents + + self.detObj = self.args.detObj + + self.threshold = None + if self.args.threshold is not None: + try: + self.threshold = eval(self.args.threshold) + except Exception as e: + print("Error evaluating threshold: " + str(e)) + logger.exception("Error evaluating threshold: " + str(e)) + self.threshold = None + + self.seedCut = None + if self.args.seedCut is not None: + try: + self.seedCut = eval(self.args.seedCut) + except Exception as e: + print("Error evaluating seedcut: " + str(e)) + logger.exception("Error evaluating seedcut: " + str(e)) + self.seedCut = None + + self.fluxCutMin = self.args.fluxCutMin + self.fluxCutMax = self.args.fluxCutMax + + self.runRange = None + if self.args.runRange is not None: + try: + self.runRange = eval(self.args.runRange) ## in case needed + except Exception as e: + print("Error evaluating runRange: " + str(e)) + logger.exception("Error evaluating runRange: " + str(e)) + self.runRange = None + + self.loadPedestalGainOffsetFiles() + + if self.args.detType == "": + ## assume epix10k for now + if self.args.nModules is not None: + self.detectorInfo.setNModules(self.args.nModules) + self.detType = self.detectorInfo.getCameraType() + else: + self.detType = self.args.detType + + self.analyzedModules = self.experimentHash.get("analyzedModules", None) + if self.analyzedModules is not None: + try: + self.analyzedModules = range(self.detectorInfo.nModules) + except Exception as e: + print("Error evaluating range: " + str(e)) + logger.info("Error evaluating range: " + str(e)) + + self.g0cut = self.detectorInfo.g0cut + if self.g0cut is not None: + self.gainBitsMask = self.g0cut - 1 + else: + self.gainBitsMask = 0xFFFF ## might be dumb. for non-autoranging + + self.negativeGain = self.detectorInfo.negativeGain ## could just use the detector info in places it's defined + + def loadPedestalGainOffsetFiles(self): + """ + Loads the pedestal, gain, and offset np files for analysis. + The files-locations are specified with cmd-line arguments. + """ + + self.fivePedestalRun = self.args.fivePedestalRun ## in case needed + + self.fakePedestal = None + self.fakePedestalFile = self.args.fakePedestalFile + if self.fakePedestalFile is not None: + try: + self.fakePedestal = np.load(self.fakePedestalFile) ##cast to uint32??? + except Exception as e: + print("Error loading fake pedistal: " + str(e)) + logger.exception("Error loading fake pedistal: " + str(e)) + + self.g0PedFile = self.args.g0PedFile + if self.g0PedFile is not None: + ##self.g0Ped = np.load(self.g0PedFile) + self.g0Ped = np.array([np.load(self.g0PedFile)]) ##temp hack + print(self.g0Ped.shape) + + self.g1PedFile = self.args.g0PedFile + if self.g1PedFile is not None: + ##self.g1Ped = np.load(self.g1PedFile) + self.g1Ped = np.array([np.load(self.g1PedFile)]) ##temp hack + + self.g0GainFile = self.args.g0GainFile + if self.g0GainFile is not None: + self.g0Gain = np.load(self.g0GainFile) + + self.g1GainFile = self.args.g1GainFile + if self.g1GainFile is not None: + self.g1Gain = np.load(self.g1GainFile) + + self.offsetFile = self.args.offsetFile + if self.offsetFile is not None: + self.offset = np.load(self.offsetFile) + + def setupOutputDirString(self, analysisType): + """ + Sets up the output directory for saving output file (default dir-name is based on the analysis type). + The output directory can be specified via a command-line argument, and its location must be specified relative to the + 'OUTPUT_ROOT environment variable. If the directory does not exist, it logs error and exits. + """ + + # output dir is where we dump .npy, .h5, and .png files + self.outputDir = "/%s/" % (analysisType) + if self.args.path is not None: + self.outputDir = self.args.path + # if set, output folders will be relative to OUTPUT_ROOT + # if not, they will be relative to the current script file + self.outputDir = os.getenv("OUTPUT_ROOT", ".") + self.outputDir + # check if outputDir exists, if does not create it and tell user + if not os.path.exists(self.outputDir): + print("could not find output dir: " + self.outputDir) + logger.info("could not find output dir: " + self.outputDir) + print("please create this dir, exiting...") + logger.info("please create this dir, exiting...") + exit(1) + # the following doesnt work with mpi parallelism (other thread could make dir b4 curr thread) + # print("so creating dir: " + self.outputDir) + # logger.info("creating dir: " + self.outputDir) + # os.makedirs(self.outputDir) + # give dir read, write, execute permissions + # os.chmod(self.outputDir, 0o777) + else: + print("output dir: " + self.outputDir) + logger.info("output dir: " + self.outputDir) + + #### End of setup related functions #### diff --git a/suite_scripts/AnalyzeH5.py b/suite_scripts/AnalyzeH5.py index cb9f4b9..2aad755 100644 --- a/suite_scripts/AnalyzeH5.py +++ b/suite_scripts/AnalyzeH5.py @@ -7,20 +7,20 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## +import logging +import os + import h5py +import matplotlib.pyplot as plt import numpy as np -import calibrationSuite.fitFunctions as fitFunctions -import calibrationSuite.ancillaryMethods as ancillaryMethods -import matplotlib.pyplot as plt +import calibrationSuite.ancillaryMethods as ancillaryMethods +import calibrationSuite.fitFunctions as fitFunctions +import calibrationSuite.loggingSetup as ls ##import sys from calibrationSuite.argumentParser import ArgumentParser -import logging -import calibrationSuite.loggingSetup as ls -import os - # log to file named .log currFileName = os.path.basename(__file__) ls.setupScriptLogging(currFileName[:-3] + ".log", logging.ERROR) # change to logging.INFO for full logging output diff --git a/suite_scripts/CalcNoiseAndMean.py b/suite_scripts/CalcNoiseAndMean.py index 537a04e..0123895 100755 --- a/suite_scripts/CalcNoiseAndMean.py +++ b/suite_scripts/CalcNoiseAndMean.py @@ -10,8 +10,9 @@ import logging import os -import calibrationSuite.loggingSetup as ls import numpy as np + +import calibrationSuite.loggingSetup as ls from calibrationSuite.basicSuiteScript import BasicSuiteScript from calibrationSuite.Stats import Stats diff --git a/suite_scripts/DeltaTtest.py b/suite_scripts/DeltaTtest.py index 834429c..9abc8b2 100644 --- a/suite_scripts/DeltaTtest.py +++ b/suite_scripts/DeltaTtest.py @@ -8,10 +8,12 @@ ## the terms contained in the LICENSE.txt file. ############################################################################## import sys -import numpy as np + import matplotlib.pyplot as plt +import numpy as np from matplotlib.ticker import AutoMinorLocator -from calibrationSuite.basicSuiteScript import BasicSuiteScript, sortArrayByList + +from calibrationSuite.basicSuiteScript import BasicSuiteScript class EventScanParallel(BasicSuiteScript): @@ -131,8 +133,8 @@ def analyze_h5(self, dataFile, label): np.save("%s/pulseIds_c%d_r%d_%s.npy" % (self.outputDir, self.camera, self.run, self.exp), np.array(pulseIds)) dPulseId = pulseIds[1:] - pulseIds[0:-1] - pixels = sortArrayByList(ts, pixels) - rois = sortArrayByList(ts, rois) + pixels = self.sortArrayByList(ts, pixels) + rois = self.sortArrayByList(ts, rois) ts.sort() ts = ts - ts[0] ##ts = ts/np.median(ts[1:]-ts[0:-1]) diff --git a/suite_scripts/EventScanParallelSlice.py b/suite_scripts/EventScanParallelSlice.py index 3da0d69..98387cd 100644 --- a/suite_scripts/EventScanParallelSlice.py +++ b/suite_scripts/EventScanParallelSlice.py @@ -7,18 +7,17 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## +import logging import os import sys -import logging -import numpy as np +import h5py import matplotlib.pyplot as plt +import numpy as np from matplotlib.ticker import AutoMinorLocator -import h5py - -from calibrationSuite.basicSuiteScript import BasicSuiteScript, sortArrayByList import calibrationSuite.loggingSetup as ls +from calibrationSuite.basicSuiteScript import BasicSuiteScript # for logging from current file logger = logging.getLogger(__name__) @@ -181,9 +180,9 @@ def analyze_h5(self, dataFile, label): dPulseId = pulseIds[1:] - pulseIds[0:-1] # sort pixels and rois based on timestamps - pixels = sortArrayByList(ts, pixels) + pixels = self.sortArrayByList(ts, pixels) if rois is not None: - rois = sortArrayByList(ts, rois) + rois = self.sortArrayByList(ts, rois) ts.sort() ts = ts - ts[0] diff --git a/suite_scripts/LinearityPlotsParallelSlice.py b/suite_scripts/LinearityPlotsParallelSlice.py index 35cd981..9bd5489 100755 --- a/suite_scripts/LinearityPlotsParallelSlice.py +++ b/suite_scripts/LinearityPlotsParallelSlice.py @@ -11,12 +11,13 @@ import os import sys -import calibrationSuite.ancillaryMethods as ancillaryMethods -import calibrationSuite.fitFunctions as fitFunctions -import calibrationSuite.loggingSetup as ls import h5py import matplotlib.pyplot as plt import numpy as np + +import calibrationSuite.ancillaryMethods as ancillaryMethods +import calibrationSuite.fitFunctions as fitFunctions +import calibrationSuite.loggingSetup as ls from calibrationSuite.basicSuiteScript import BasicSuiteScript # for logging from current file @@ -217,7 +218,7 @@ def analyze_h5_slice(self, dataFile, label): iDet, jDet = self.sliceToDetector(i, j) try: self.fitInfo[module, i, j, 8] = self.g0Ped[module, iDet, jDet] - except: + except Exception: pass if False: self.fitInfo[module, i, j, 9] = self.g1Ped[module, iDet, jDet] @@ -374,8 +375,8 @@ def analyze_h5_slice(self, dataFile, label): lpp.setupPsana() try: - size = comm.Get_size() - except: + size = comm.Get_size() # noqa: F821 + except Exception: size = 1 smd = lpp.ds.smalldata( filename="%s/%s_%s_c%d_r%d_n%d.h5" % (lpp.outputDir, lpp.className, lpp.label, lpp.camera, lpp.run, size) diff --git a/suite_scripts/PersistenceCheckParallel.py b/suite_scripts/PersistenceCheckParallel.py index daac168..66cd185 100755 --- a/suite_scripts/PersistenceCheckParallel.py +++ b/suite_scripts/PersistenceCheckParallel.py @@ -7,11 +7,13 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## -from calibrationSuite.basicSuiteScript import BasicSuiteScript -import h5py import sys -import numpy as np + +import h5py import matplotlib.pyplot as plt +import numpy as np + +from calibrationSuite.basicSuiteScript import BasicSuiteScript class PersistenceCheckParallel(BasicSuiteScript): diff --git a/suite_scripts/SimpleClustersParallelSlice.py b/suite_scripts/SimpleClustersParallelSlice.py index 7e2fe37..4220762 100644 --- a/suite_scripts/SimpleClustersParallelSlice.py +++ b/suite_scripts/SimpleClustersParallelSlice.py @@ -9,12 +9,13 @@ ############################################################################## import sys -import calibrationSuite.fitFunctions as fitFunctions import matplotlib.pyplot as plt import numpy as np +from scipy.optimize import curve_fit + +import calibrationSuite.fitFunctions as fitFunctions from calibrationSuite.basicSuiteScript import BasicSuiteScript from calibrationSuite.cluster import BuildClusters -from scipy.optimize import curve_fit class SimpleClusters(BasicSuiteScript): diff --git a/suite_scripts/TimeScanParallelSlice.py b/suite_scripts/TimeScanParallelSlice.py index 3cfecf9..26bf0a7 100755 --- a/suite_scripts/TimeScanParallelSlice.py +++ b/suite_scripts/TimeScanParallelSlice.py @@ -11,13 +11,14 @@ import os import sys -import calibrationSuite.loggingSetup as ls import h5py import matplotlib.pyplot as plt import numpy as np -from calibrationSuite.basicSuiteScript import BasicSuiteScript from matplotlib.ticker import AutoMinorLocator +import calibrationSuite.loggingSetup as ls +from calibrationSuite.basicSuiteScript import BasicSuiteScript + # for logging from current file logger = logging.getLogger(__name__) # log to file named .log diff --git a/suite_scripts/findMinSwitchValue.py b/suite_scripts/findMinSwitchValue.py index 20b4b86..3cd3f50 100755 --- a/suite_scripts/findMinSwitchValue.py +++ b/suite_scripts/findMinSwitchValue.py @@ -9,6 +9,7 @@ ############################################################################## import matplotlib.pyplot as plt import numpy as np + from calibrationSuite.basicSuiteScript import BasicSuiteScript diff --git a/suite_scripts/histogramFluxEtc.py b/suite_scripts/histogramFluxEtc.py index 5c41fcc..0e47f46 100755 --- a/suite_scripts/histogramFluxEtc.py +++ b/suite_scripts/histogramFluxEtc.py @@ -9,6 +9,7 @@ ############################################################################## import matplotlib.pyplot as plt import numpy as np + from calibrationSuite.basicSuiteScript import BasicSuiteScript diff --git a/suite_scripts/persistenceCheck.py b/suite_scripts/persistenceCheck.py index 7eaef11..c768b97 100755 --- a/suite_scripts/persistenceCheck.py +++ b/suite_scripts/persistenceCheck.py @@ -7,10 +7,12 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## -from calibrationSuite.basicSuiteScript import BasicSuiteScript import sys -import numpy as np + import matplotlib.pyplot as plt +import numpy as np + +from calibrationSuite.basicSuiteScript import BasicSuiteScript class PersistenceCheck(BasicSuiteScript): diff --git a/suite_scripts/roiFromSwitched.py b/suite_scripts/roiFromSwitched.py index d06431a..47bd44e 100755 --- a/suite_scripts/roiFromSwitched.py +++ b/suite_scripts/roiFromSwitched.py @@ -10,8 +10,9 @@ import logging import os -import calibrationSuite.loggingSetup as ls import numpy as np + +import calibrationSuite.loggingSetup as ls from calibrationSuite.basicSuiteScript import BasicSuiteScript # for logging from current file diff --git a/suite_scripts/roiFromThreshold.py b/suite_scripts/roiFromThreshold.py index 5dc9356..8bc373b 100755 --- a/suite_scripts/roiFromThreshold.py +++ b/suite_scripts/roiFromThreshold.py @@ -10,6 +10,7 @@ import sys import numpy as np + from calibrationSuite.basicSuiteScript import BasicSuiteScript diff --git a/suite_scripts/runAnalyzeH5.py b/suite_scripts/runAnalyzeH5.py index 6d6c6a1..a28b276 100644 --- a/suite_scripts/runAnalyzeH5.py +++ b/suite_scripts/runAnalyzeH5.py @@ -7,8 +7,8 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## -import sys import glob +import sys basePath = sys.argv[1] baseRun = sys.argv[2] diff --git a/suite_scripts/searchForNonSwitching.py b/suite_scripts/searchForNonSwitching.py index ab6b738..9c9358e 100755 --- a/suite_scripts/searchForNonSwitching.py +++ b/suite_scripts/searchForNonSwitching.py @@ -8,6 +8,7 @@ ## the terms contained in the LICENSE.txt file. ############################################################################## import matplotlib.pyplot as plt + from calibrationSuite.basicSuiteScript import BasicSuiteScript diff --git a/suite_scripts/simplePhotonCounter.py b/suite_scripts/simplePhotonCounter.py index 2eb9b16..78882cc 100755 --- a/suite_scripts/simplePhotonCounter.py +++ b/suite_scripts/simplePhotonCounter.py @@ -10,8 +10,9 @@ import logging import os -import calibrationSuite.loggingSetup as ls import numpy as np + +import calibrationSuite.loggingSetup as ls from calibrationSuite.basicSuiteScript import BasicSuiteScript # for logging from current file diff --git a/tests/test_AncillaryMethods.py b/tests/test_AncillaryMethods.py index 1e38e79..c94bc15 100644 --- a/tests/test_AncillaryMethods.py +++ b/tests/test_AncillaryMethods.py @@ -38,30 +38,22 @@ def test_getEnergeticClusters(): def test_getSmallSquareClusters(): - clusters = np.array( - [[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]] - ) + clusters = np.array([[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]]) small_square_clusters = getSmallSquareClusters(clusters) assert np.array_equal(small_square_clusters, [[2, 1, 2, 2, 3, 1]]) def test_getSmallClusters(): - clusters = np.array( - [[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]] - ) + clusters = np.array([[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]]) small_clusters = getSmallClusters(clusters) assert np.array_equal(small_clusters, [[2, 1, 2, 2, 3, 1]]) def test_getSquareClusters(): - clusters = np.array( - [[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]] - ) + clusters = np.array([[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]]) square_clusters = getSquareClusters(clusters) - assert np.array_equal( - square_clusters, [[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [3, 1, 2, 2, 6, 1]] - ) + assert np.array_equal(square_clusters, [[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [3, 1, 2, 2, 6, 1]]) def test_getMatchedClusters(): @@ -79,9 +71,7 @@ def test_getMatchedClusters(): dimension = "column" n = 6 matched_clusters = getMatchedClusters(clusters, dimension, n) - assert np.array_equal( - matched_clusters, [[1, 1, 1, 6], [2, 1, 2, 6], [2, 2, 2, 6], [3, 4, 3, 6]] - ) + assert np.array_equal(matched_clusters, [[1, 1, 1, 6], [2, 1, 2, 6], [2, 2, 2, 6], [3, 4, 3, 6]]) def test_goodClusters(): diff --git a/tests/test_ArgumentParser.py b/tests/test_ArgumentParser.py index cf92c8d..aa14976 100644 --- a/tests/test_ArgumentParser.py +++ b/tests/test_ArgumentParser.py @@ -80,9 +80,7 @@ def test_parsing_4(parser): # Check: python SimpleClustersParallelSlice.py -r 224 -f ../lowFlux/SimpleClusters_c0_r224_n100.h5 def test_parsing_5(parser): - args = parser.parse_args( - ["-r", "224", "-f", "../lowFlux/SimpleClusters_c0_r224_n100.h5"] - ) + args = parser.parse_args(["-r", "224", "-f", "../lowFlux/SimpleClusters_c0_r224_n100.h5"]) assert args.run == 224 assert args.files == "../lowFlux/SimpleClusters_c0_r224_n100.h5" assert args.label is None diff --git a/tests/test_BasicSuiteScript.py b/tests/test_BasicSuiteScript.py index cccd3c3..005e783 100644 --- a/tests/test_BasicSuiteScript.py +++ b/tests/test_BasicSuiteScript.py @@ -1,6 +1,7 @@ import os -import sys import subprocess +import sys + import pytest # Quick and simple test that creates the main calibrationSuite object that all the /suite_scripts @@ -11,6 +12,7 @@ def psana_installed(): try: import psana # noqa: F401 + return True except ImportError: return False @@ -28,10 +30,17 @@ def test_example(psana_installed): # annoyingly complicated way to get root of current git repo, # do this so test can be run from tests/ dir or root of project - git_repo_root = (subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE).communicate()[0].rstrip().decode("utf-8")) + git_repo_root = ( + subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) + .communicate()[0] + .rstrip() + .decode("utf-8") + ) os.environ["OUTPUT_ROOT"] = "." os.environ["SUITE_CONFIG"] = git_repo_root + "/tests/testingSuiteConfig.py" + # to be sure this script acts the same as those in suite_scripts (in regards to file-paths) + os.chdir(git_repo_root + "/suite_scripts/") os.makedirs("basic_suite_test", exist_ok=True) bSS = BasicSuiteScript() @@ -47,7 +56,7 @@ def test_example(psana_installed): stepGen = bSS.getStepGen() assert stepGen is not None - + # this is slow, don't do for now """ count = 0 @@ -65,7 +74,6 @@ def test_example(psana_installed): # TODO: add checks that init and setupPsana stuff is set properly, # also call and test directly some class-methods. - """ # also here's a way to potentially test creating a base object base = PsanaBase() diff --git a/tests/test_Clusters.py b/tests/test_Clusters.py index ea74384..40d66ce 100644 --- a/tests/test_Clusters.py +++ b/tests/test_Clusters.py @@ -49,6 +49,4 @@ def test_build_clusters_find_clusters(): assert clusters[0].eTotalNoCuts == 10 assert clusters[0].eSecondaryPixelNoCuts == 0 assert clusters[0].goodCluster is True - assert np.array_equal( - clusters[0].pixelE, np.array([[0, 0, 0], [0, 10, 0], [0, 0, 0]]) - ) + assert np.array_equal(clusters[0].pixelE, np.array([[0, 0, 0], [0, 10, 0], [0, 0, 0]])) diff --git a/tests/test_DetectorInfo.py b/tests/test_DetectorInfo.py index d1d3a25..ca123eb 100644 --- a/tests/test_DetectorInfo.py +++ b/tests/test_DetectorInfo.py @@ -39,13 +39,27 @@ def test_epixm_setup(): assert detector.neighborCut == 0.25 -def test_archon_setup(): +def test_archon_setup_1d(): detector = DetectorInfo("archon") assert detector.detectorType == "archon" + assert detector.dimension == 2 assert detector.nTestPixelsPerBank == 36 assert detector.nBanks == 16 assert detector.nCols == 4800 - detector.nBanks * detector.nTestPixelsPerBank assert detector.nRows == 1 assert detector.preferredCommonMode == "rixsCCDTestPixelSubtraction" assert detector.clusterShape == [1, 5] + + +def test_archon_setup_2d(): + detector = DetectorInfo("archon", "2d") + + assert detector.detectorType == "archon" + assert detector.dimension == 3 + assert detector.nTestPixelsPerBank == 36 + assert detector.nBanks == 16 + assert detector.nCols == 4800 - detector.nBanks * detector.nTestPixelsPerBank + assert detector.nRows == 1200 + assert detector.preferredCommonMode == "rixsCCDTestPixelSubtraction" + assert detector.clusterShape == [3, 5] diff --git a/tests/test_FitFunctions.py b/tests/test_FitFunctions.py index b114d95..03163f5 100644 --- a/tests/test_FitFunctions.py +++ b/tests/test_FitFunctions.py @@ -24,9 +24,7 @@ def test_linear(a, b, expected): np.testing.assert_array_equal(result, expected) -@pytest.mark.parametrize( - "a, b, c, d, expected", [(2, 3, 4, 10, np.array([5, 7, 9, 10, 10]))] -) +@pytest.mark.parametrize("a, b, c, d, expected", [(2, 3, 4, 10, np.array([5, 7, 9, 10, 10]))]) def test_saturatedLinear(a, b, c, d, expected): x = np.array([1, 2, 3, 4, 5]) result = saturatedLinear(x, a, b, c, d) @@ -73,9 +71,7 @@ def test_gaussianArea(a, sigma, expected): def test_estimateGaussianParametersFromUnbinnedArray(): flatData = np.array([1, 2, 3, 4, 5]) - result_amp, result_mean, result_sigma = estimateGaussianParametersFromUnbinnedArray( - flatData - ) + result_amp, result_mean, result_sigma = estimateGaussianParametersFromUnbinnedArray(flatData) expected_amp, expected_mean, expected_sigma = ( 0.5629831060402448, 3.0, @@ -109,9 +105,7 @@ def test_getHistogramMeanStd(): assert np.isclose(result_std, expected_std) -@pytest.mark.parametrize( - "y, fit, expected_r2", [(np.array([1, 2, 3, 4, 5]), np.array([1, 2, 3, 4, 6]), 0.9)] -) +@pytest.mark.parametrize("y, fit, expected_r2", [(np.array([1, 2, 3, 4, 5]), np.array([1, 2, 3, 4, 6]), 0.9)]) def test_calculateFitR2(y, fit, expected_r2): result_r2 = calculateFitR2(y, fit) print(result_r2) diff --git a/tests/test_MiscTests.py b/tests/test_MiscTests.py index 7db370e..dfe39a1 100644 --- a/tests/test_MiscTests.py +++ b/tests/test_MiscTests.py @@ -1,15 +1,18 @@ -import os import re +import subprocess def test_environment_setup(): - # so we can run from project root or 'tests' folder - curr_dir = os.getcwd() - setup_script_path = "" - if "tests" in curr_dir: - setup_script_path = "../setup.sh" - else: - setup_script_path = "setup.sh" + # annoyingly complicated way to get root of current git repo, + # do this so test can be run from tests/ dir or root of project + git_repo_root = ( + subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) + .communicate()[0] + .rstrip() + .decode("utf-8") + ) + + setup_script_path = git_repo_root + "/setup.sh" # Read the script to extract expected values expected_values = {} diff --git a/tests/test_Stats.py b/tests/test_Stats.py index f8e476b..fa35998 100644 --- a/tests/test_Stats.py +++ b/tests/test_Stats.py @@ -47,9 +47,7 @@ def test_Stats_rms(): expected_xx += d * d calculated_rms = s.rms() - expected_rms = (expected_xx / expected_n - (expected_x / expected_n) ** 2).clip( - 0 - ) ** 0.5 + expected_rms = (expected_xx / expected_n - (expected_x / expected_n) ** 2).clip(0) ** 0.5 assert np.isclose(calculated_rms[0], expected_rms), "RMS calculation incorrect" @@ -78,6 +76,4 @@ def test_Stats_corr(): calculated_corr = s.corr(y_mean, y_sigma) expected_corr = np.corrcoef(x_data, y_data)[0, 1] - assert np.isclose( - calculated_corr[0], expected_corr - ), "Correlation calculation incorrect" + assert np.isclose(calculated_corr[0], expected_corr), "Correlation calculation incorrect" diff --git a/tests/test_SuiteScripts.py b/tests/test_SuiteScripts.py index d550f6a..326c403 100644 --- a/tests/test_SuiteScripts.py +++ b/tests/test_SuiteScripts.py @@ -84,14 +84,15 @@ def __init__(self): # annoyingly complicated way to get root of current git repo, # do this so test can be run from tests/ dir or root of project self.git_repo_root = ( - subprocess.Popen( - ["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE - ) + subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) .communicate()[0] .rstrip() .decode("utf-8") ) + # avoid any weirdness and make sure curr-dir is that of tested scripts + os.chdir(self.git_repo_root + "/suite_scripts") + # tests can only run if the following are true (skip if not): # 1) pasna library is avaliable (i.e running on S3DF) # 2) tests/test-data submodule is installed @@ -130,9 +131,7 @@ def __init__(self): # lets have the 'real output' folders just be in /suite_scripts for i in range(len(self.expected_outcome_dirs)): - self.expected_outcome_dirs[i] = ( - self.git_repo_root + "/suite_scripts/" + self.expected_outcome_dirs[i] - ) + self.expected_outcome_dirs[i] = self.git_repo_root + "/suite_scripts/" + self.expected_outcome_dirs[i] for dir in self.expected_outcome_dirs: os.makedirs(dir, exist_ok=True) @@ -173,8 +172,8 @@ def test_command(self, command, output_location): if result.returncode != 0: assert False, f"Script failed with error: {result.stderr}" - real_output_location = "../suite_scripts/" + output_location - expected_output_location = "./test_data/" + output_location + real_output_location = self.git_repo_root + "/suite_scripts/" + output_location + expected_output_location = self.git_repo_root + "/tests/test_data/" + output_location # Compare files in directories for root, dirs, files in os.walk(real_output_location): @@ -183,9 +182,7 @@ def test_command(self, command, output_location): expected_file_path = expected_output_location + "/" + file # Check if files are PNGs - if real_file_path.endswith(".png") and expected_file_path.endswith( - ".png" - ): + if real_file_path.endswith(".png") and expected_file_path.endswith(".png"): if self.are_images_same(real_file_path, expected_file_path) == 0: assert False, f"PNG files {real_file_path} and {expected_file_path} are different" else: @@ -330,7 +327,7 @@ def test_SinglePhoton(suite_tester, command, output_dir_name): "bash", "-c", # this cmd runs pretty long, so we use '--special testing' and '-maxNevents 2' to stop pixel-analysis early - "python LinearityPlotsParallelSlice.py -r 102 --special testing --maxNevents 2 -p /test_linearity_scan -f test_linearity_scan/LinearityPlotsParallel__c0_r102_n666.h5 --label fooBar", + "python LinearityPlotsParallelSlice.py -r 102 --special testing --maxNevents 2 -p /test_linearity_scan -f test_linearity_scan/LinearityPlotsParallel__c0_r102_n1.h5 --label fooBar", ], "test_linearity_scan", ), diff --git a/tests/test_data b/tests/test_data index a0f0d08..13b3223 160000 --- a/tests/test_data +++ b/tests/test_data @@ -1 +1 @@ -Subproject commit a0f0d08766613ee934d137dcc934df286854df8b +Subproject commit 13b3223222b517e983199cf8d6f4657f8333c00e