From 9649d4a982ab4ce07f5b0078bdceba92dcc05dbd Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Tue, 18 Jun 2024 22:56:12 -0700 Subject: [PATCH 01/16] MNT: apply linting-formatting script and fix test failures previous introduced --- calibrationSuite/basicSuiteScript.py | 4 ++-- suite_scripts/AnalyzeH5.py | 14 +++++++------- suite_scripts/CalcNoiseAndMean.py | 3 ++- suite_scripts/DeltaTtest.py | 4 +++- suite_scripts/EventScanParallelSlice.py | 9 ++++----- suite_scripts/LinearityPlotsParallelSlice.py | 13 +++++++------ suite_scripts/PersistenceCheckParallel.py | 8 +++++--- suite_scripts/SimpleClustersParallelSlice.py | 5 +++-- suite_scripts/TimeScanParallelSlice.py | 5 +++-- suite_scripts/findMinSwitchValue.py | 1 + suite_scripts/histogramFluxEtc.py | 1 + suite_scripts/persistenceCheck.py | 6 ++++-- suite_scripts/roiFromSwitched.py | 3 ++- suite_scripts/roiFromThreshold.py | 1 + suite_scripts/runAnalyzeH5.py | 2 +- suite_scripts/searchForNonSwitching.py | 1 + suite_scripts/simplePhotonCounter.py | 3 ++- tests/test_AncillaryMethods.py | 20 +++++--------------- tests/test_ArgumentParser.py | 4 +--- tests/test_BasicSuiteScript.py | 14 ++++++++++---- tests/test_Clusters.py | 4 +--- tests/test_FitFunctions.py | 12 +++--------- tests/test_Stats.py | 8 ++------ tests/test_SuiteScripts.py | 14 ++++---------- tests/test_data | 2 +- 25 files changed, 76 insertions(+), 85 deletions(-) diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index 29b2417..99b7c43 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -18,10 +18,10 @@ if os.getenv("foo") == "1": print("psana1") - from calibrationSuite.psana1Base import PsanaBase + from calibrationSuite.psana1Base import PsanaBase, comm else: print("psana2") - from calibrationSuite.psana2Base import PsanaBase + from calibrationSuite.psana2Base import PsanaBase, comm def sortArrayByList(a, data): diff --git a/suite_scripts/AnalyzeH5.py b/suite_scripts/AnalyzeH5.py index cb9f4b9..2aad755 100644 --- a/suite_scripts/AnalyzeH5.py +++ b/suite_scripts/AnalyzeH5.py @@ -7,20 +7,20 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## +import logging +import os + import h5py +import matplotlib.pyplot as plt import numpy as np -import calibrationSuite.fitFunctions as fitFunctions -import calibrationSuite.ancillaryMethods as ancillaryMethods -import matplotlib.pyplot as plt +import calibrationSuite.ancillaryMethods as ancillaryMethods +import calibrationSuite.fitFunctions as fitFunctions +import calibrationSuite.loggingSetup as ls ##import sys from calibrationSuite.argumentParser import ArgumentParser -import logging -import calibrationSuite.loggingSetup as ls -import os - # log to file named .log currFileName = os.path.basename(__file__) ls.setupScriptLogging(currFileName[:-3] + ".log", logging.ERROR) # change to logging.INFO for full logging output diff --git a/suite_scripts/CalcNoiseAndMean.py b/suite_scripts/CalcNoiseAndMean.py index 537a04e..0123895 100755 --- a/suite_scripts/CalcNoiseAndMean.py +++ b/suite_scripts/CalcNoiseAndMean.py @@ -10,8 +10,9 @@ import logging import os -import calibrationSuite.loggingSetup as ls import numpy as np + +import calibrationSuite.loggingSetup as ls from calibrationSuite.basicSuiteScript import BasicSuiteScript from calibrationSuite.Stats import Stats diff --git a/suite_scripts/DeltaTtest.py b/suite_scripts/DeltaTtest.py index 834429c..cbf2589 100644 --- a/suite_scripts/DeltaTtest.py +++ b/suite_scripts/DeltaTtest.py @@ -8,9 +8,11 @@ ## the terms contained in the LICENSE.txt file. ############################################################################## import sys -import numpy as np + import matplotlib.pyplot as plt +import numpy as np from matplotlib.ticker import AutoMinorLocator + from calibrationSuite.basicSuiteScript import BasicSuiteScript, sortArrayByList diff --git a/suite_scripts/EventScanParallelSlice.py b/suite_scripts/EventScanParallelSlice.py index 3da0d69..a12de58 100644 --- a/suite_scripts/EventScanParallelSlice.py +++ b/suite_scripts/EventScanParallelSlice.py @@ -7,18 +7,17 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## +import logging import os import sys -import logging -import numpy as np +import h5py import matplotlib.pyplot as plt +import numpy as np from matplotlib.ticker import AutoMinorLocator -import h5py - -from calibrationSuite.basicSuiteScript import BasicSuiteScript, sortArrayByList import calibrationSuite.loggingSetup as ls +from calibrationSuite.basicSuiteScript import BasicSuiteScript, sortArrayByList # for logging from current file logger = logging.getLogger(__name__) diff --git a/suite_scripts/LinearityPlotsParallelSlice.py b/suite_scripts/LinearityPlotsParallelSlice.py index 35cd981..4a6f8fc 100755 --- a/suite_scripts/LinearityPlotsParallelSlice.py +++ b/suite_scripts/LinearityPlotsParallelSlice.py @@ -11,13 +11,14 @@ import os import sys -import calibrationSuite.ancillaryMethods as ancillaryMethods -import calibrationSuite.fitFunctions as fitFunctions -import calibrationSuite.loggingSetup as ls import h5py import matplotlib.pyplot as plt import numpy as np -from calibrationSuite.basicSuiteScript import BasicSuiteScript + +import calibrationSuite.ancillaryMethods as ancillaryMethods +import calibrationSuite.fitFunctions as fitFunctions +import calibrationSuite.loggingSetup as ls +from calibrationSuite.basicSuiteScript import BasicSuiteScript, comm # for logging from current file logger = logging.getLogger(__name__) @@ -217,7 +218,7 @@ def analyze_h5_slice(self, dataFile, label): iDet, jDet = self.sliceToDetector(i, j) try: self.fitInfo[module, i, j, 8] = self.g0Ped[module, iDet, jDet] - except: + except Exception: pass if False: self.fitInfo[module, i, j, 9] = self.g1Ped[module, iDet, jDet] @@ -375,7 +376,7 @@ def analyze_h5_slice(self, dataFile, label): try: size = comm.Get_size() - except: + except Exception: size = 1 smd = lpp.ds.smalldata( filename="%s/%s_%s_c%d_r%d_n%d.h5" % (lpp.outputDir, lpp.className, lpp.label, lpp.camera, lpp.run, size) diff --git a/suite_scripts/PersistenceCheckParallel.py b/suite_scripts/PersistenceCheckParallel.py index daac168..66cd185 100755 --- a/suite_scripts/PersistenceCheckParallel.py +++ b/suite_scripts/PersistenceCheckParallel.py @@ -7,11 +7,13 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## -from calibrationSuite.basicSuiteScript import BasicSuiteScript -import h5py import sys -import numpy as np + +import h5py import matplotlib.pyplot as plt +import numpy as np + +from calibrationSuite.basicSuiteScript import BasicSuiteScript class PersistenceCheckParallel(BasicSuiteScript): diff --git a/suite_scripts/SimpleClustersParallelSlice.py b/suite_scripts/SimpleClustersParallelSlice.py index 7e2fe37..4220762 100644 --- a/suite_scripts/SimpleClustersParallelSlice.py +++ b/suite_scripts/SimpleClustersParallelSlice.py @@ -9,12 +9,13 @@ ############################################################################## import sys -import calibrationSuite.fitFunctions as fitFunctions import matplotlib.pyplot as plt import numpy as np +from scipy.optimize import curve_fit + +import calibrationSuite.fitFunctions as fitFunctions from calibrationSuite.basicSuiteScript import BasicSuiteScript from calibrationSuite.cluster import BuildClusters -from scipy.optimize import curve_fit class SimpleClusters(BasicSuiteScript): diff --git a/suite_scripts/TimeScanParallelSlice.py b/suite_scripts/TimeScanParallelSlice.py index 3cfecf9..26bf0a7 100755 --- a/suite_scripts/TimeScanParallelSlice.py +++ b/suite_scripts/TimeScanParallelSlice.py @@ -11,13 +11,14 @@ import os import sys -import calibrationSuite.loggingSetup as ls import h5py import matplotlib.pyplot as plt import numpy as np -from calibrationSuite.basicSuiteScript import BasicSuiteScript from matplotlib.ticker import AutoMinorLocator +import calibrationSuite.loggingSetup as ls +from calibrationSuite.basicSuiteScript import BasicSuiteScript + # for logging from current file logger = logging.getLogger(__name__) # log to file named .log diff --git a/suite_scripts/findMinSwitchValue.py b/suite_scripts/findMinSwitchValue.py index 20b4b86..3cd3f50 100755 --- a/suite_scripts/findMinSwitchValue.py +++ b/suite_scripts/findMinSwitchValue.py @@ -9,6 +9,7 @@ ############################################################################## import matplotlib.pyplot as plt import numpy as np + from calibrationSuite.basicSuiteScript import BasicSuiteScript diff --git a/suite_scripts/histogramFluxEtc.py b/suite_scripts/histogramFluxEtc.py index 5c41fcc..0e47f46 100755 --- a/suite_scripts/histogramFluxEtc.py +++ b/suite_scripts/histogramFluxEtc.py @@ -9,6 +9,7 @@ ############################################################################## import matplotlib.pyplot as plt import numpy as np + from calibrationSuite.basicSuiteScript import BasicSuiteScript diff --git a/suite_scripts/persistenceCheck.py b/suite_scripts/persistenceCheck.py index 7eaef11..c768b97 100755 --- a/suite_scripts/persistenceCheck.py +++ b/suite_scripts/persistenceCheck.py @@ -7,10 +7,12 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## -from calibrationSuite.basicSuiteScript import BasicSuiteScript import sys -import numpy as np + import matplotlib.pyplot as plt +import numpy as np + +from calibrationSuite.basicSuiteScript import BasicSuiteScript class PersistenceCheck(BasicSuiteScript): diff --git a/suite_scripts/roiFromSwitched.py b/suite_scripts/roiFromSwitched.py index d06431a..47bd44e 100755 --- a/suite_scripts/roiFromSwitched.py +++ b/suite_scripts/roiFromSwitched.py @@ -10,8 +10,9 @@ import logging import os -import calibrationSuite.loggingSetup as ls import numpy as np + +import calibrationSuite.loggingSetup as ls from calibrationSuite.basicSuiteScript import BasicSuiteScript # for logging from current file diff --git a/suite_scripts/roiFromThreshold.py b/suite_scripts/roiFromThreshold.py index 5dc9356..8bc373b 100755 --- a/suite_scripts/roiFromThreshold.py +++ b/suite_scripts/roiFromThreshold.py @@ -10,6 +10,7 @@ import sys import numpy as np + from calibrationSuite.basicSuiteScript import BasicSuiteScript diff --git a/suite_scripts/runAnalyzeH5.py b/suite_scripts/runAnalyzeH5.py index 6d6c6a1..a28b276 100644 --- a/suite_scripts/runAnalyzeH5.py +++ b/suite_scripts/runAnalyzeH5.py @@ -7,8 +7,8 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## -import sys import glob +import sys basePath = sys.argv[1] baseRun = sys.argv[2] diff --git a/suite_scripts/searchForNonSwitching.py b/suite_scripts/searchForNonSwitching.py index ab6b738..9c9358e 100755 --- a/suite_scripts/searchForNonSwitching.py +++ b/suite_scripts/searchForNonSwitching.py @@ -8,6 +8,7 @@ ## the terms contained in the LICENSE.txt file. ############################################################################## import matplotlib.pyplot as plt + from calibrationSuite.basicSuiteScript import BasicSuiteScript diff --git a/suite_scripts/simplePhotonCounter.py b/suite_scripts/simplePhotonCounter.py index 2eb9b16..78882cc 100755 --- a/suite_scripts/simplePhotonCounter.py +++ b/suite_scripts/simplePhotonCounter.py @@ -10,8 +10,9 @@ import logging import os -import calibrationSuite.loggingSetup as ls import numpy as np + +import calibrationSuite.loggingSetup as ls from calibrationSuite.basicSuiteScript import BasicSuiteScript # for logging from current file diff --git a/tests/test_AncillaryMethods.py b/tests/test_AncillaryMethods.py index 1e38e79..c94bc15 100644 --- a/tests/test_AncillaryMethods.py +++ b/tests/test_AncillaryMethods.py @@ -38,30 +38,22 @@ def test_getEnergeticClusters(): def test_getSmallSquareClusters(): - clusters = np.array( - [[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]] - ) + clusters = np.array([[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]]) small_square_clusters = getSmallSquareClusters(clusters) assert np.array_equal(small_square_clusters, [[2, 1, 2, 2, 3, 1]]) def test_getSmallClusters(): - clusters = np.array( - [[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]] - ) + clusters = np.array([[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]]) small_clusters = getSmallClusters(clusters) assert np.array_equal(small_clusters, [[2, 1, 2, 2, 3, 1]]) def test_getSquareClusters(): - clusters = np.array( - [[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]] - ) + clusters = np.array([[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [2, 1, 2, 2, 5, 0], [3, 1, 2, 2, 6, 1]]) square_clusters = getSquareClusters(clusters) - assert np.array_equal( - square_clusters, [[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [3, 1, 2, 2, 6, 1]] - ) + assert np.array_equal(square_clusters, [[1, 1, 1, 1, 4, 1], [2, 1, 2, 2, 3, 1], [3, 1, 2, 2, 6, 1]]) def test_getMatchedClusters(): @@ -79,9 +71,7 @@ def test_getMatchedClusters(): dimension = "column" n = 6 matched_clusters = getMatchedClusters(clusters, dimension, n) - assert np.array_equal( - matched_clusters, [[1, 1, 1, 6], [2, 1, 2, 6], [2, 2, 2, 6], [3, 4, 3, 6]] - ) + assert np.array_equal(matched_clusters, [[1, 1, 1, 6], [2, 1, 2, 6], [2, 2, 2, 6], [3, 4, 3, 6]]) def test_goodClusters(): diff --git a/tests/test_ArgumentParser.py b/tests/test_ArgumentParser.py index cf92c8d..aa14976 100644 --- a/tests/test_ArgumentParser.py +++ b/tests/test_ArgumentParser.py @@ -80,9 +80,7 @@ def test_parsing_4(parser): # Check: python SimpleClustersParallelSlice.py -r 224 -f ../lowFlux/SimpleClusters_c0_r224_n100.h5 def test_parsing_5(parser): - args = parser.parse_args( - ["-r", "224", "-f", "../lowFlux/SimpleClusters_c0_r224_n100.h5"] - ) + args = parser.parse_args(["-r", "224", "-f", "../lowFlux/SimpleClusters_c0_r224_n100.h5"]) assert args.run == 224 assert args.files == "../lowFlux/SimpleClusters_c0_r224_n100.h5" assert args.label is None diff --git a/tests/test_BasicSuiteScript.py b/tests/test_BasicSuiteScript.py index cccd3c3..ced606d 100644 --- a/tests/test_BasicSuiteScript.py +++ b/tests/test_BasicSuiteScript.py @@ -1,6 +1,7 @@ import os -import sys import subprocess +import sys + import pytest # Quick and simple test that creates the main calibrationSuite object that all the /suite_scripts @@ -11,6 +12,7 @@ def psana_installed(): try: import psana # noqa: F401 + return True except ImportError: return False @@ -28,7 +30,12 @@ def test_example(psana_installed): # annoyingly complicated way to get root of current git repo, # do this so test can be run from tests/ dir or root of project - git_repo_root = (subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE).communicate()[0].rstrip().decode("utf-8")) + git_repo_root = ( + subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) + .communicate()[0] + .rstrip() + .decode("utf-8") + ) os.environ["OUTPUT_ROOT"] = "." os.environ["SUITE_CONFIG"] = git_repo_root + "/tests/testingSuiteConfig.py" @@ -47,7 +54,7 @@ def test_example(psana_installed): stepGen = bSS.getStepGen() assert stepGen is not None - + # this is slow, don't do for now """ count = 0 @@ -65,7 +72,6 @@ def test_example(psana_installed): # TODO: add checks that init and setupPsana stuff is set properly, # also call and test directly some class-methods. - """ # also here's a way to potentially test creating a base object base = PsanaBase() diff --git a/tests/test_Clusters.py b/tests/test_Clusters.py index ea74384..40d66ce 100644 --- a/tests/test_Clusters.py +++ b/tests/test_Clusters.py @@ -49,6 +49,4 @@ def test_build_clusters_find_clusters(): assert clusters[0].eTotalNoCuts == 10 assert clusters[0].eSecondaryPixelNoCuts == 0 assert clusters[0].goodCluster is True - assert np.array_equal( - clusters[0].pixelE, np.array([[0, 0, 0], [0, 10, 0], [0, 0, 0]]) - ) + assert np.array_equal(clusters[0].pixelE, np.array([[0, 0, 0], [0, 10, 0], [0, 0, 0]])) diff --git a/tests/test_FitFunctions.py b/tests/test_FitFunctions.py index b114d95..03163f5 100644 --- a/tests/test_FitFunctions.py +++ b/tests/test_FitFunctions.py @@ -24,9 +24,7 @@ def test_linear(a, b, expected): np.testing.assert_array_equal(result, expected) -@pytest.mark.parametrize( - "a, b, c, d, expected", [(2, 3, 4, 10, np.array([5, 7, 9, 10, 10]))] -) +@pytest.mark.parametrize("a, b, c, d, expected", [(2, 3, 4, 10, np.array([5, 7, 9, 10, 10]))]) def test_saturatedLinear(a, b, c, d, expected): x = np.array([1, 2, 3, 4, 5]) result = saturatedLinear(x, a, b, c, d) @@ -73,9 +71,7 @@ def test_gaussianArea(a, sigma, expected): def test_estimateGaussianParametersFromUnbinnedArray(): flatData = np.array([1, 2, 3, 4, 5]) - result_amp, result_mean, result_sigma = estimateGaussianParametersFromUnbinnedArray( - flatData - ) + result_amp, result_mean, result_sigma = estimateGaussianParametersFromUnbinnedArray(flatData) expected_amp, expected_mean, expected_sigma = ( 0.5629831060402448, 3.0, @@ -109,9 +105,7 @@ def test_getHistogramMeanStd(): assert np.isclose(result_std, expected_std) -@pytest.mark.parametrize( - "y, fit, expected_r2", [(np.array([1, 2, 3, 4, 5]), np.array([1, 2, 3, 4, 6]), 0.9)] -) +@pytest.mark.parametrize("y, fit, expected_r2", [(np.array([1, 2, 3, 4, 5]), np.array([1, 2, 3, 4, 6]), 0.9)]) def test_calculateFitR2(y, fit, expected_r2): result_r2 = calculateFitR2(y, fit) print(result_r2) diff --git a/tests/test_Stats.py b/tests/test_Stats.py index f8e476b..fa35998 100644 --- a/tests/test_Stats.py +++ b/tests/test_Stats.py @@ -47,9 +47,7 @@ def test_Stats_rms(): expected_xx += d * d calculated_rms = s.rms() - expected_rms = (expected_xx / expected_n - (expected_x / expected_n) ** 2).clip( - 0 - ) ** 0.5 + expected_rms = (expected_xx / expected_n - (expected_x / expected_n) ** 2).clip(0) ** 0.5 assert np.isclose(calculated_rms[0], expected_rms), "RMS calculation incorrect" @@ -78,6 +76,4 @@ def test_Stats_corr(): calculated_corr = s.corr(y_mean, y_sigma) expected_corr = np.corrcoef(x_data, y_data)[0, 1] - assert np.isclose( - calculated_corr[0], expected_corr - ), "Correlation calculation incorrect" + assert np.isclose(calculated_corr[0], expected_corr), "Correlation calculation incorrect" diff --git a/tests/test_SuiteScripts.py b/tests/test_SuiteScripts.py index d550f6a..a46ebed 100644 --- a/tests/test_SuiteScripts.py +++ b/tests/test_SuiteScripts.py @@ -84,9 +84,7 @@ def __init__(self): # annoyingly complicated way to get root of current git repo, # do this so test can be run from tests/ dir or root of project self.git_repo_root = ( - subprocess.Popen( - ["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE - ) + subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) .communicate()[0] .rstrip() .decode("utf-8") @@ -130,9 +128,7 @@ def __init__(self): # lets have the 'real output' folders just be in /suite_scripts for i in range(len(self.expected_outcome_dirs)): - self.expected_outcome_dirs[i] = ( - self.git_repo_root + "/suite_scripts/" + self.expected_outcome_dirs[i] - ) + self.expected_outcome_dirs[i] = self.git_repo_root + "/suite_scripts/" + self.expected_outcome_dirs[i] for dir in self.expected_outcome_dirs: os.makedirs(dir, exist_ok=True) @@ -183,9 +179,7 @@ def test_command(self, command, output_location): expected_file_path = expected_output_location + "/" + file # Check if files are PNGs - if real_file_path.endswith(".png") and expected_file_path.endswith( - ".png" - ): + if real_file_path.endswith(".png") and expected_file_path.endswith(".png"): if self.are_images_same(real_file_path, expected_file_path) == 0: assert False, f"PNG files {real_file_path} and {expected_file_path} are different" else: @@ -330,7 +324,7 @@ def test_SinglePhoton(suite_tester, command, output_dir_name): "bash", "-c", # this cmd runs pretty long, so we use '--special testing' and '-maxNevents 2' to stop pixel-analysis early - "python LinearityPlotsParallelSlice.py -r 102 --special testing --maxNevents 2 -p /test_linearity_scan -f test_linearity_scan/LinearityPlotsParallel__c0_r102_n666.h5 --label fooBar", + "python LinearityPlotsParallelSlice.py -r 102 --special testing --maxNevents 2 -p /test_linearity_scan -f test_linearity_scan/LinearityPlotsParallel__c0_r102_n1.h5 --label fooBar", ], "test_linearity_scan", ), diff --git a/tests/test_data b/tests/test_data index a0f0d08..a6ac5ab 160000 --- a/tests/test_data +++ b/tests/test_data @@ -1 +1 @@ -Subproject commit a0f0d08766613ee934d137dcc934df286854df8b +Subproject commit a6ac5ab0f3e448f5a5f3d9e6ba80455b9647de29 From 27754cfd78fca4bb6672f53a2307f8d08273aa74 Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Wed, 19 Jun 2024 13:54:49 -0700 Subject: [PATCH 02/16] TST: fix detector info tests, add new test for 1d vs 2d --- calibrationSuite/detectorInfo.py | 4 ++-- tests/test_DetectorInfo.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/calibrationSuite/detectorInfo.py b/calibrationSuite/detectorInfo.py index f48edd5..2347543 100644 --- a/calibrationSuite/detectorInfo.py +++ b/calibrationSuite/detectorInfo.py @@ -8,7 +8,7 @@ ## the terms contained in the LICENSE.txt file. ############################################################################## class DetectorInfo: - def __init__(self, detType, detSubtype=None): + def __init__(self, detType, detSubtype="1d"): # declare all detector-specific info vars here in case any setup_X functions don't, # and use -1 so caller knows things are not setup (non-0 to avoid error on divide. self.nModules = -1 @@ -88,7 +88,7 @@ def setup_epixM(self, version=0): self.seedCut = 2 self.neighborCut = 0.25 ## ditto - def setup_rixsCCD(self, mode="1d", version=0): + def setup_rixsCCD(self, mode, version=0): print("rixsCCD mode:", mode) self.nTestPixelsPerBank = 36 self.nBanks = 16 diff --git a/tests/test_DetectorInfo.py b/tests/test_DetectorInfo.py index d1d3a25..ca123eb 100644 --- a/tests/test_DetectorInfo.py +++ b/tests/test_DetectorInfo.py @@ -39,13 +39,27 @@ def test_epixm_setup(): assert detector.neighborCut == 0.25 -def test_archon_setup(): +def test_archon_setup_1d(): detector = DetectorInfo("archon") assert detector.detectorType == "archon" + assert detector.dimension == 2 assert detector.nTestPixelsPerBank == 36 assert detector.nBanks == 16 assert detector.nCols == 4800 - detector.nBanks * detector.nTestPixelsPerBank assert detector.nRows == 1 assert detector.preferredCommonMode == "rixsCCDTestPixelSubtraction" assert detector.clusterShape == [1, 5] + + +def test_archon_setup_2d(): + detector = DetectorInfo("archon", "2d") + + assert detector.detectorType == "archon" + assert detector.dimension == 3 + assert detector.nTestPixelsPerBank == 36 + assert detector.nBanks == 16 + assert detector.nCols == 4800 - detector.nBanks * detector.nTestPixelsPerBank + assert detector.nRows == 1200 + assert detector.preferredCommonMode == "rixsCCDTestPixelSubtraction" + assert detector.clusterShape == [3, 5] From 60dc36a6019ca65b7eef81548015236c2ea128d8 Mon Sep 17 00:00:00 2001 From: nstelter Date: Wed, 19 Jun 2024 14:37:40 -0700 Subject: [PATCH 03/16] WIP: work on lib code cleanup --- calibrationSuite/basicSuiteScript.py | 98 ++++++++++------------------ calibrationSuite/psana1Base.py | 14 +--- calibrationSuite/psana2Base.py | 16 +---- 3 files changed, 36 insertions(+), 92 deletions(-) diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index 99b7c43..ed6d5f3 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -31,6 +31,7 @@ def sortArrayByList(a, data): class BasicSuiteScript(PsanaBase): def __init__(self, analysisType="scan"): super().__init__() + print("in BasicSuiteScript, inheriting from PsanaBase, type is psana%d" % (self.psanaType)) logger.info("in BasicSuiteScript, inheriting from PsanaBase, type is psana%d" % (self.psanaType)) @@ -38,51 +39,35 @@ def __init__(self, analysisType="scan"): self.ePix10k_cameraTypes = {1: "Epix10ka", 4: "Epix10kaQuad", 16: "Epix10ka2M"} self.camera = 0 - self.outputDir = "/%s/" % (analysisType) - logger.info("output dir: " + self.outputDir) - print("output dir: " + self.outputDir) - try: self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"], self.experimentHash["detectorSubtype"]) except: self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"]) + self.className = self.__class__.__name__ - try: - self.location = self.experimentHash["location"] - except Exception: - pass - try: - self.exp = self.experimentHash["exp"] - except Exception: - pass + self.exp = self.experimentHash.get("exp", None) self.ROIfileNames = None + try: - ##if True: - self.ROIfileNames = self.experimentHash["ROIs"] + self.ROIfileNames = self.experimentHash.get("ROIs", None) self.ROIs = [] for f in self.ROIfileNames: self.ROIs.append(np.load(f)) - try: ## dumb code for compatibility or expectation - self.ROI = self.ROIs[0] - except Exception: - pass - ##if False: + self.ROI = self.ROIs[0] if len(self.ROIs) >= 1 else None except Exception: if self.ROIfileNames is not None: - print("had trouble finding", self.ROIfileNames) + print("had trouble finding" + str(self.ROIfileNames)) + logger.error("had trouble finding" + str(self.ROIfileNames)) for currName in self.ROIfileNames: + print("had trouble finding" + currName) logger.exception("had trouble finding" + currName) - self.ROI = None - self.ROIs = [] - try: - self.singlePixels = self.experimentHash["singlePixels"] - except Exception: - self.singlePixels = None - try: - self.regionSlice = self.experimentHash["regionSlice"] - except Exception: - self.regionSlice = None + # exit to make clear to user things went wrong... + exit(1) + + self.singlePixels = self.experimentHash.get("singlePixels", None) + + self.regionSlice = self.experimentHash.get("regionSlice", None) if self.regionSlice is not None: ## n.b. assumes 3d slice now self.sliceCoordinates = [ @@ -96,28 +81,18 @@ def __init__(self, analysisType="scan"): if self.detectorInfo.dimension == 2: self.regionSlice = self.regionSlice[0], self.regionSlice[2] print("remapping regionSlice to handle 1d case") + logger.info("remapping regionSlice to handle 1d case") - try: - self.fluxSource = self.experimentHash["fluxSource"] - try: - self.fluxChannels = self.experimentHash["fluxChannels"] - except Exception: - self.fluxChannels = range(8, 16) ## wave8 - try: - self.fluxSign = self.experimentHash["fluxSign"] - except Exception: - self.fluxSign = 1 - except Exception: - self.fluxSource = None - try: - self.ignoreEventCodeCheck = self.experimentHash["ignoreEventCodeCheck"] - self.fakeBeamCode = True ## just in case - except Exception: - self.ignoreEventCodeCheck = False - self.fakeBeamCode = False + self.fluxSource = self.experimentHash.get("fluxSource", None) + self.fluxChannels = self.experimentHas.get("fluxChannels", range(8, 16)) ## wave8 + self.fluxSign = self.experimentHash.get("fluxSign", 1) + + self.ignoreEventCodeCheck = self.experimentHash("ignoreEventCodeCheck", None) + self.fakeBeamCode = True if self.ignoreEventCodeCheck is not None else False self.special = self.args.special + ## for non-120 Hz running self.nRunCodeEvents = 0 self.nDaqCodeEvents = 0 @@ -130,9 +105,8 @@ def __init__(self, analysisType="scan"): if self.special is not None: self.fakeBeamCode = "fakeBeamCode" in self.special - print("ignoring event code check, faking beam code:", self.ignoreEventCodeCheck, self.fakeBeamCode) - - ##mymodule = importlib.import_module(full_module_name) + print("ignoring event code check, faking beam code:" + str(self.ignoreEventCodeCheck) + " " + str(self.fakeBeamCode)) + logger.info("ignoring event code check, faking beam code:" + str(self.ignoreEventCodeCheck) + " " + str(self.fakeBeamCode)) ## for standalone analysis self.file = None @@ -149,12 +123,16 @@ def __init__(self, analysisType="scan"): self.camera = self.args.camera if self.args.exp is not None: self.exp = self.args.exp + + self.location = self.experimentHash.get("location", None) if self.args.location is not None: self.location = self.args.location if self.args.maxNevents is not None: self.maxNevents = self.args.maxNevents if self.args.skipNevents is not None: self.skipNevents = self.args.skipNevents + + self.outputDir = "/%s/" % (analysisType) if self.args.path is not None: self.outputDir = self.args.path # if set, output folders will be relative to OUTPUT_ROOT @@ -173,6 +151,10 @@ def __init__(self, analysisType="scan"): # os.makedirs(self.outputDir) # give dir read, write, execute permissions # os.chmod(self.outputDir, 0o777) + else: + print("output dir: " + self.outputDir) + logger.info("output dir: " + self.outputDir) + self.detObj = self.args.detObj if self.args.threshold is not None: self.threshold = eval(self.args.threshold) @@ -393,16 +375,4 @@ def addFakePhotons(self, frames, occupancy, E, width): return frames + fakes, (fakes > 0).sum() def getNswitchedPixels(self, data, region=None): - return ((data >= self.g0cut) * 1).sum() - - -""" -if __name__ == "__main__": - bSS = BasicSuiteScript() - print("have built a BasicSuiteScript") - logger.info("have built a BasicSuiteScript") - bSS.setupPsana() - evt = bSS.getEvt() - print(dir(evt)) - logger.info(dir(evt)) -""" + return ((data >= self.g0cut) * 1).sum() \ No newline at end of file diff --git a/calibrationSuite/psana1Base.py b/calibrationSuite/psana1Base.py index d010b2e..6eaf286 100755 --- a/calibrationSuite/psana1Base.py +++ b/calibrationSuite/psana1Base.py @@ -28,8 +28,6 @@ def __init__(self, analysisType="scan"): self.psanaType = 1 print("in psana1Base") logger.info("in psana1Base") - self.gainModes = {"FH": 0, "FM": 1, "FL": 2, "AHL-H": 3, "AML-M": 4, "AHL-L": 5, "AML-L": 6} - self.ePix10k_cameraTypes = {1: "Epix10ka", 4: "Epix10kaQuad", 16: "Epix10ka2M"} self.g0cut = 1 << 14 self.gainBitsMask = self.g0cut - 1 @@ -216,14 +214,4 @@ def getCalibData(self, evt): return self.det.calib(evt) def getImage(self, evt, data=None): - return self.raw.image(evt, data) - - -""" -if __name__ == "__main__": - bSS = BasicSuiteScript() - print("have built a BasicSuiteScript") - bSS.setupPsana() - evt = bSS.getEvt() - print(dir(evt)) -""" + return self.raw.image(evt, data) \ No newline at end of file diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index 9900b93..592c704 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -47,10 +47,6 @@ def __init__(self, analysisType="scan"): print("in psana2Base") logger.info("in psana2Base") - self.gainModes = {"FH": 0, "FM": 1, "FL": 2, "AHL-H": 3, "AML-M": 4, "AHL-L": 5, "AML-L": 6} - self.ePix10k_cameraTypes = {1: "Epix10ka", 4: "Epix10kaQuad", 16: "Epix10ka2M"} - ##self.g0cut = 1<<15 ## 2022 - self.allowed_timestamp_mismatch = 1000 self.args = ArgumentParser().parse_args() @@ -337,14 +333,4 @@ def getPingPongParity(self, frameRegion): oddsEvenRowsEvensOddRows = frameRegion[1::2, ::2] + frameRegion[::2, 1::2] delta = evensEvenRowsOddsOddRows.mean() - oddsEvenRowsEvensOddRows.mean() ##print("delta:", delta) - return delta > 0 - - -""" -if __name__ == "__main__": - bSS = BasicSuiteScript() - print("have built a BasicSuiteScript") - bSS.setupPsana() - evt = bSS.getEvt() - print(dir(evt)) -""" + return delta > 0 \ No newline at end of file From 451c98a2d24a24d8afc8a44ef49ad4e067e23b96 Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Wed, 19 Jun 2024 17:01:43 -0700 Subject: [PATCH 04/16] WIP: improving test's path to resources to be absolute so can run anywehre --- calibrationSuite/basicSuiteScript.py | 33 +++++++++++++------- calibrationSuite/psana1Base.py | 2 +- calibrationSuite/psana2Base.py | 2 +- suite_scripts/LinearityPlotsParallelSlice.py | 4 +-- tests/test_BasicSuiteScript.py | 2 ++ tests/test_MiscTests.py | 19 ++++++----- tests/test_SuiteScripts.py | 7 +++-- 7 files changed, 44 insertions(+), 25 deletions(-) diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index ed6d5f3..1dd72fc 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -18,10 +18,10 @@ if os.getenv("foo") == "1": print("psana1") - from calibrationSuite.psana1Base import PsanaBase, comm + from calibrationSuite.psana1Base import PsanaBase else: print("psana2") - from calibrationSuite.psana2Base import PsanaBase, comm + from calibrationSuite.psana2Base import PsanaBase def sortArrayByList(a, data): @@ -40,10 +40,12 @@ def __init__(self, analysisType="scan"): self.camera = 0 try: - self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"], self.experimentHash["detectorSubtype"]) - except: + self.detectorInfo = DetectorInfo( + self.experimentHash["detectorType"], self.experimentHash["detectorSubtype"] + ) + except Exception: self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"]) - + self.className = self.__class__.__name__ self.exp = self.experimentHash.get("exp", None) @@ -83,12 +85,11 @@ def __init__(self, analysisType="scan"): print("remapping regionSlice to handle 1d case") logger.info("remapping regionSlice to handle 1d case") - self.fluxSource = self.experimentHash.get("fluxSource", None) - self.fluxChannels = self.experimentHas.get("fluxChannels", range(8, 16)) ## wave8 + self.fluxChannels = self.experimentHash.get("fluxChannels", range(8, 16)) ## wave8 self.fluxSign = self.experimentHash.get("fluxSign", 1) - self.ignoreEventCodeCheck = self.experimentHash("ignoreEventCodeCheck", None) + self.ignoreEventCodeCheck = self.experimentHash.get("ignoreEventCodeCheck", None) self.fakeBeamCode = True if self.ignoreEventCodeCheck is not None else False self.special = self.args.special @@ -105,8 +106,18 @@ def __init__(self, analysisType="scan"): if self.special is not None: self.fakeBeamCode = "fakeBeamCode" in self.special - print("ignoring event code check, faking beam code:" + str(self.ignoreEventCodeCheck) + " " + str(self.fakeBeamCode)) - logger.info("ignoring event code check, faking beam code:" + str(self.ignoreEventCodeCheck) + " " + str(self.fakeBeamCode)) + print( + "ignoring event code check, faking beam code:" + + str(self.ignoreEventCodeCheck) + + " " + + str(self.fakeBeamCode) + ) + logger.info( + "ignoring event code check, faking beam code:" + + str(self.ignoreEventCodeCheck) + + " " + + str(self.fakeBeamCode) + ) ## for standalone analysis self.file = None @@ -375,4 +386,4 @@ def addFakePhotons(self, frames, occupancy, E, width): return frames + fakes, (fakes > 0).sum() def getNswitchedPixels(self, data, region=None): - return ((data >= self.g0cut) * 1).sum() \ No newline at end of file + return ((data >= self.g0cut) * 1).sum() diff --git a/calibrationSuite/psana1Base.py b/calibrationSuite/psana1Base.py index 6eaf286..af286ef 100755 --- a/calibrationSuite/psana1Base.py +++ b/calibrationSuite/psana1Base.py @@ -214,4 +214,4 @@ def getCalibData(self, evt): return self.det.calib(evt) def getImage(self, evt, data=None): - return self.raw.image(evt, data) \ No newline at end of file + return self.raw.image(evt, data) diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index 592c704..faf8a9e 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -333,4 +333,4 @@ def getPingPongParity(self, frameRegion): oddsEvenRowsEvensOddRows = frameRegion[1::2, ::2] + frameRegion[::2, 1::2] delta = evensEvenRowsOddsOddRows.mean() - oddsEvenRowsEvensOddRows.mean() ##print("delta:", delta) - return delta > 0 \ No newline at end of file + return delta > 0 diff --git a/suite_scripts/LinearityPlotsParallelSlice.py b/suite_scripts/LinearityPlotsParallelSlice.py index 4a6f8fc..9bd5489 100755 --- a/suite_scripts/LinearityPlotsParallelSlice.py +++ b/suite_scripts/LinearityPlotsParallelSlice.py @@ -18,7 +18,7 @@ import calibrationSuite.ancillaryMethods as ancillaryMethods import calibrationSuite.fitFunctions as fitFunctions import calibrationSuite.loggingSetup as ls -from calibrationSuite.basicSuiteScript import BasicSuiteScript, comm +from calibrationSuite.basicSuiteScript import BasicSuiteScript # for logging from current file logger = logging.getLogger(__name__) @@ -375,7 +375,7 @@ def analyze_h5_slice(self, dataFile, label): lpp.setupPsana() try: - size = comm.Get_size() + size = comm.Get_size() # noqa: F821 except Exception: size = 1 smd = lpp.ds.smalldata( diff --git a/tests/test_BasicSuiteScript.py b/tests/test_BasicSuiteScript.py index ced606d..005e783 100644 --- a/tests/test_BasicSuiteScript.py +++ b/tests/test_BasicSuiteScript.py @@ -39,6 +39,8 @@ def test_example(psana_installed): os.environ["OUTPUT_ROOT"] = "." os.environ["SUITE_CONFIG"] = git_repo_root + "/tests/testingSuiteConfig.py" + # to be sure this script acts the same as those in suite_scripts (in regards to file-paths) + os.chdir(git_repo_root + "/suite_scripts/") os.makedirs("basic_suite_test", exist_ok=True) bSS = BasicSuiteScript() diff --git a/tests/test_MiscTests.py b/tests/test_MiscTests.py index 7db370e..dfe39a1 100644 --- a/tests/test_MiscTests.py +++ b/tests/test_MiscTests.py @@ -1,15 +1,18 @@ -import os import re +import subprocess def test_environment_setup(): - # so we can run from project root or 'tests' folder - curr_dir = os.getcwd() - setup_script_path = "" - if "tests" in curr_dir: - setup_script_path = "../setup.sh" - else: - setup_script_path = "setup.sh" + # annoyingly complicated way to get root of current git repo, + # do this so test can be run from tests/ dir or root of project + git_repo_root = ( + subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) + .communicate()[0] + .rstrip() + .decode("utf-8") + ) + + setup_script_path = git_repo_root + "/setup.sh" # Read the script to extract expected values expected_values = {} diff --git a/tests/test_SuiteScripts.py b/tests/test_SuiteScripts.py index a46ebed..326c403 100644 --- a/tests/test_SuiteScripts.py +++ b/tests/test_SuiteScripts.py @@ -90,6 +90,9 @@ def __init__(self): .decode("utf-8") ) + # avoid any weirdness and make sure curr-dir is that of tested scripts + os.chdir(self.git_repo_root + "/suite_scripts") + # tests can only run if the following are true (skip if not): # 1) pasna library is avaliable (i.e running on S3DF) # 2) tests/test-data submodule is installed @@ -169,8 +172,8 @@ def test_command(self, command, output_location): if result.returncode != 0: assert False, f"Script failed with error: {result.stderr}" - real_output_location = "../suite_scripts/" + output_location - expected_output_location = "./test_data/" + output_location + real_output_location = self.git_repo_root + "/suite_scripts/" + output_location + expected_output_location = self.git_repo_root + "/tests/test_data/" + output_location # Compare files in directories for root, dirs, files in os.walk(real_output_location): From f0ac15bc2e7d7ed1575f32274e159a03ef8e1f5d Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Wed, 19 Jun 2024 18:47:34 -0700 Subject: [PATCH 05/16] WIP: refactoring lib code --- calibrationSuite/basicSuiteScript.py | 172 +++++++++++++++------------ calibrationSuite/psana2Base.py | 1 + 2 files changed, 96 insertions(+), 77 deletions(-) diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index 1dd72fc..a0bcfd2 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -34,11 +34,28 @@ def __init__(self, analysisType="scan"): print("in BasicSuiteScript, inheriting from PsanaBase, type is psana%d" % (self.psanaType)) logger.info("in BasicSuiteScript, inheriting from PsanaBase, type is psana%d" % (self.psanaType)) + self.className = self.__class__.__name__ self.gainModes = {"FH": 0, "FM": 1, "FL": 2, "AHL-H": 3, "AML-M": 4, "AHL-L": 5, "AML-L": 6} self.ePix10k_cameraTypes = {1: "Epix10ka", 4: "Epix10kaQuad", 16: "Epix10ka2M"} - self.camera = 0 + self.ds = None + self.det = None ## do we need multiple dets in an array? or self.secondDet? + + ## for non-120 Hz running + self.nRunCodeEvents = 0 + self.nDaqCodeEvents = 0 + self.nBeamCodeEvents = 0 + self.runCode = 280 + self.daqCode = 281 + self.beamCode = 283 ## per Matt + ##self.beamCode = 281 ## don't see 283... + + self.setupFromExperimentHash() + self.setupFromCmdlineArgs() + self.setupOutputDirString(analysisType) + + def setupFromExperimentHash(self): try: self.detectorInfo = DetectorInfo( self.experimentHash["detectorType"], self.experimentHash["detectorSubtype"] @@ -46,8 +63,6 @@ def __init__(self, analysisType="scan"): except Exception: self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"]) - self.className = self.__class__.__name__ - self.exp = self.experimentHash.get("exp", None) self.ROIfileNames = None @@ -64,8 +79,6 @@ def __init__(self, analysisType="scan"): for currName in self.ROIfileNames: print("had trouble finding" + currName) logger.exception("had trouble finding" + currName) - # exit to make clear to user things went wrong... - exit(1) self.singlePixels = self.experimentHash.get("singlePixels", None) @@ -92,16 +105,9 @@ def __init__(self, analysisType="scan"): self.ignoreEventCodeCheck = self.experimentHash.get("ignoreEventCodeCheck", None) self.fakeBeamCode = True if self.ignoreEventCodeCheck is not None else False + def setupFromCmdlineArgs(self): self.special = self.args.special - ## for non-120 Hz running - self.nRunCodeEvents = 0 - self.nDaqCodeEvents = 0 - self.nBeamCodeEvents = 0 - self.runCode = 280 - self.daqCode = 281 - self.beamCode = 283 ## per Matt - ##self.beamCode = 281 ## don't see 283... if not self.fakeBeamCode: ## defined in ignoreEventCodeCheck if self.special is not None: self.fakeBeamCode = "fakeBeamCode" in self.special @@ -120,75 +126,67 @@ def __init__(self, analysisType="scan"): ) ## for standalone analysis - self.file = None - if self.args.files is not None: - self.file = self.args.files - self.label = "" - if self.args.label is not None: - self.label = self.args.label + self.file = self.args.files + self.label = "" if self.args.label is None else self.args.label ## analyzing xtcs - if self.args.run is not None: - self.run = self.args.run - if self.args.camera is not None: - self.camera = self.args.camera + self.run = self.args.run + + self.camera = 0 if self.args.camera is None else self.args.camera + + # this is set in the config-file, but take the cmd-line value instead if it is set if self.args.exp is not None: self.exp = self.args.exp self.location = self.experimentHash.get("location", None) if self.args.location is not None: self.location = self.args.location - if self.args.maxNevents is not None: - self.maxNevents = self.args.maxNevents - if self.args.skipNevents is not None: - self.skipNevents = self.args.skipNevents - self.outputDir = "/%s/" % (analysisType) - if self.args.path is not None: - self.outputDir = self.args.path - # if set, output folders will be relative to OUTPUT_ROOT - # if not, they will be relative to the current script file - self.outputDir = os.getenv("OUTPUT_ROOT", ".") + self.outputDir - # check if outputDir exists, if does not create it and tell user - if not os.path.exists(self.outputDir): - print("could not find output dir: " + self.outputDir) - logger.info("could not find output dir: " + self.outputDir) - print("please create this dir, exiting...") - logger.info("please create this dir, exiting...") - exit(1) - # the following doesnt work with mpi parallelism (other thread could make dir b4 curr thread) - # print("so creating dir: " + self.outputDir) - # logger.info("creating dir: " + self.outputDir) - # os.makedirs(self.outputDir) - # give dir read, write, execute permissions - # os.chmod(self.outputDir, 0o777) - else: - print("output dir: " + self.outputDir) - logger.info("output dir: " + self.outputDir) + self.maxNevents = self.args.maxNevents + self.skipNevents = self.args.skipNevents self.detObj = self.args.detObj + + self.threshold = None if self.args.threshold is not None: - self.threshold = eval(self.args.threshold) - else: - self.threshold = None + try: + self.threshold = eval(self.args.threshold) + except Exception as e: + print("Error evaluating threshold: " + str(e)) + logger.exception("Error evaluating threshold: " + str(e)) + self.threshold = None + + self.seedCut = None if self.args.seedCut is not None: - self.seedCut = eval(self.args.seedCut) - else: - self.seedCut = None - if self.args.fluxCutMin is not None: - self.fluxCutMin = self.args.fluxCutMin - if self.args.fluxCutMax is not None: - self.fluxCutMax = self.args.fluxCutMax - try: - self.runRange = eval(self.args.runRange) ## in case needed - except Exception: - self.runRange = None + try: + self.seedCut = eval(self.args.seedCut) + except Exception as e: + print("Error evaluating seedcut: " + str(e)) + logger.exception("Error evaluating seedcut: " + str(e)) + self.seedCut = None + + self.fluxCutMin = self.args.fluxCutMin + self.fluxCutMax = self.args.fluxCutMax + + self.runRange = None + if self.args.runRange is not None: + try: + self.runRange = eval(self.args.runRange) ## in case needed + except Exception as e: + print("Error evaluating runRange: " + str(e)) + logger.exception("Error evaluating runRange: " + str(e)) + self.runRange = None self.fivePedestalRun = self.args.fivePedestalRun ## in case needed + self.fakePedestal = None - self.fakePedestalFile = self.args.fakePedestalFile ## in case needed + self.fakePedestalFile = self.args.fakePedestalFile if self.fakePedestalFile is not None: - self.fakePedestal = np.load(self.fakePedestalFile) ##cast to uint32??? + try: + self.fakePedestal = np.load(self.fakePedestalFile) ##cast to uint32??? + except Exception as e: + print("Error loading fake pedistal: " + str(e)) + logger.exception("Error loading fake pedistal: " + str(e)) self.g0PedFile = self.args.g0PedFile if self.g0PedFile is not None: @@ -221,26 +219,46 @@ def __init__(self, analysisType="scan"): else: self.detType = self.args.detType - try: - self.analyzedModules = self.experimentHash["analyzedModules"] - except Exception: - self.analyzedModules = range(self.detectorInfo.nModules) + self.analyzedModules = self.experimentHash.get("analyzedModules", None) + if self.analyzedModules is not None: + try: + self.analyzedModules = range(self.detectorInfo.nModules) + except Exception as e: + print("Error evaluating range: " + str(e)) + logger.info("Error evaluating range: " + str(e)) self.g0cut = self.detectorInfo.g0cut if self.g0cut is not None: self.gainBitsMask = self.g0cut - 1 else: - self.gainBitsMask = 0xFFFF ## might be dumb. for non-autoranging + self.gainBitsMask = 0xFFFF ## might be dumb. for non-autoranging self.negativeGain = self.detectorInfo.negativeGain ## could just use the detector info in places it's defined - ## done with configuration - - self.ds = None - self.det = None ## do we need multiple dets in an array? or self.secondDet? - - ##self.setupPsana() - ##do this later or skip for -file + def setupOutputDirString(self, analysisType): + # setup output-dir for dumping output .npy, .h5, and .png files + self.outputDir = "/%s/" % (analysisType) + if self.args.path is not None: + self.outputDir = self.args.path + # if set, output folders will be relative to OUTPUT_ROOT + # if not, they will be relative to the current script file + self.outputDir = os.getenv("OUTPUT_ROOT", ".") + self.outputDir + # check if outputDir exists, if does not create it and tell user + if not os.path.exists(self.outputDir): + print("could not find output dir: " + self.outputDir) + logger.info("could not find output dir: " + self.outputDir) + print("please create this dir, exiting...") + logger.info("please create this dir, exiting...") + exit(1) + # the following doesnt work with mpi parallelism (other thread could make dir b4 curr thread) + # print("so creating dir: " + self.outputDir) + # logger.info("creating dir: " + self.outputDir) + # os.makedirs(self.outputDir) + # give dir read, write, execute permissions + # os.chmod(self.outputDir, 0o777) + else: + print("output dir: " + self.outputDir) + logger.info("output dir: " + self.outputDir) def setROI(self, roiFile=None, roi=None): """Call with both file name and roi to save roi to file and use, diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index faf8a9e..aef8faa 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -92,6 +92,7 @@ def get_ds(self, run=None): def setupPsana(self): ##print("have built basic script class, exp %s run %d" %(self.exp, self.run)) if self.runRange is None: + print(self.run) self.ds = self.get_ds(self.run) else: self.run = self.runRange[0] From 99c77d664d48ec2ac51a763fecd5b570b3086cb6 Mon Sep 17 00:00:00 2001 From: nstelter Date: Wed, 19 Jun 2024 18:54:07 -0700 Subject: [PATCH 06/16] WIP: more refactoring lib code --- calibrationSuite/basicSuiteScript.py | 82 +++++++++++++++------------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index a0bcfd2..f7d5fda 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -64,21 +64,8 @@ def setupFromExperimentHash(self): self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"]) self.exp = self.experimentHash.get("exp", None) - self.ROIfileNames = None - try: - self.ROIfileNames = self.experimentHash.get("ROIs", None) - self.ROIs = [] - for f in self.ROIfileNames: - self.ROIs.append(np.load(f)) - self.ROI = self.ROIs[0] if len(self.ROIs) >= 1 else None - except Exception: - if self.ROIfileNames is not None: - print("had trouble finding" + str(self.ROIfileNames)) - logger.error("had trouble finding" + str(self.ROIfileNames)) - for currName in self.ROIfileNames: - print("had trouble finding" + currName) - logger.exception("had trouble finding" + currName) + self.setupROIFiles() self.singlePixels = self.experimentHash.get("singlePixels", None) @@ -105,6 +92,22 @@ def setupFromExperimentHash(self): self.ignoreEventCodeCheck = self.experimentHash.get("ignoreEventCodeCheck", None) self.fakeBeamCode = True if self.ignoreEventCodeCheck is not None else False + def setupROIFiles(self): + self.ROIfileNames = None + try: + self.ROIfileNames = self.experimentHash.get("ROIs", None) + self.ROIs = [] + for f in self.ROIfileNames: + self.ROIs.append(np.load(f)) + self.ROI = self.ROIs[0] if len(self.ROIs) >= 1 else None + except Exception: + if self.ROIfileNames is not None: + print("had trouble finding" + str(self.ROIfileNames)) + logger.error("had trouble finding" + str(self.ROIfileNames)) + for currName in self.ROIfileNames: + print("had trouble finding" + currName) + logger.exception("had trouble finding" + currName) + def setupFromCmdlineArgs(self): self.special = self.args.special @@ -177,6 +180,33 @@ def setupFromCmdlineArgs(self): logger.exception("Error evaluating runRange: " + str(e)) self.runRange = None + self.loadPedestalGainOffsetFiles() + + if self.args.detType == "": + ## assume epix10k for now + if self.args.nModules is not None: + self.detectorInfo.setNModules(self.args.nModules) + self.detType = self.detectorInfo.getCameraType() + else: + self.detType = self.args.detType + + self.analyzedModules = self.experimentHash.get("analyzedModules", None) + if self.analyzedModules is not None: + try: + self.analyzedModules = range(self.detectorInfo.nModules) + except Exception as e: + print("Error evaluating range: " + str(e)) + logger.info("Error evaluating range: " + str(e)) + + self.g0cut = self.detectorInfo.g0cut + if self.g0cut is not None: + self.gainBitsMask = self.g0cut - 1 + else: + self.gainBitsMask = 0xFFFF ## might be dumb. for non-autoranging + + self.negativeGain = self.detectorInfo.negativeGain ## could just use the detector info in places it's defined + + def loadPedestalGainOffsetFiles(self): self.fivePedestalRun = self.args.fivePedestalRun ## in case needed self.fakePedestal = None @@ -211,30 +241,6 @@ def setupFromCmdlineArgs(self): if self.offsetFile is not None: self.offset = np.load(self.offsetFile) - if self.args.detType == "": - ## assume epix10k for now - if self.args.nModules is not None: - self.detectorInfo.setNModules(self.args.nModules) - self.detType = self.detectorInfo.getCameraType() - else: - self.detType = self.args.detType - - self.analyzedModules = self.experimentHash.get("analyzedModules", None) - if self.analyzedModules is not None: - try: - self.analyzedModules = range(self.detectorInfo.nModules) - except Exception as e: - print("Error evaluating range: " + str(e)) - logger.info("Error evaluating range: " + str(e)) - - self.g0cut = self.detectorInfo.g0cut - if self.g0cut is not None: - self.gainBitsMask = self.g0cut - 1 - else: - self.gainBitsMask = 0xFFFF ## might be dumb. for non-autoranging - - self.negativeGain = self.detectorInfo.negativeGain ## could just use the detector info in places it's defined - def setupOutputDirString(self, analysisType): # setup output-dir for dumping output .npy, .h5, and .png files self.outputDir = "/%s/" % (analysisType) From 7b2c31c99e92d6ff5786357198372b249154d0a5 Mon Sep 17 00:00:00 2001 From: nstelter Date: Wed, 19 Jun 2024 19:16:53 -0700 Subject: [PATCH 07/16] WIP: more refactoring lib code --- calibrationSuite/basicSuiteScript.py | 217 ------------------ calibrationSuite/psana1Base.py | 60 +---- calibrationSuite/psana2Base.py | 96 +------- calibrationSuite/psanaCommon.py | 328 +++++++++++++++++++++++++++ 4 files changed, 334 insertions(+), 367 deletions(-) create mode 100755 calibrationSuite/psanaCommon.py diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index f7d5fda..0ef7fae 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -12,8 +12,6 @@ import numpy as np -from calibrationSuite.detectorInfo import DetectorInfo - logger = logging.getLogger(__name__) if os.getenv("foo") == "1": @@ -51,221 +49,6 @@ def __init__(self, analysisType="scan"): self.beamCode = 283 ## per Matt ##self.beamCode = 281 ## don't see 283... - self.setupFromExperimentHash() - self.setupFromCmdlineArgs() - self.setupOutputDirString(analysisType) - - def setupFromExperimentHash(self): - try: - self.detectorInfo = DetectorInfo( - self.experimentHash["detectorType"], self.experimentHash["detectorSubtype"] - ) - except Exception: - self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"]) - - self.exp = self.experimentHash.get("exp", None) - - self.setupROIFiles() - - self.singlePixels = self.experimentHash.get("singlePixels", None) - - self.regionSlice = self.experimentHash.get("regionSlice", None) - if self.regionSlice is not None: - ## n.b. assumes 3d slice now - self.sliceCoordinates = [ - [self.regionSlice[1].start, self.regionSlice[1].stop], - [self.regionSlice[2].start, self.regionSlice[2].stop], - ] - sc = self.sliceCoordinates - self.sliceEdges = [sc[0][1] - sc[0][0], sc[1][1] - sc[1][0]] - - ## handle 1d rixs ccd data - if self.detectorInfo.dimension == 2: - self.regionSlice = self.regionSlice[0], self.regionSlice[2] - print("remapping regionSlice to handle 1d case") - logger.info("remapping regionSlice to handle 1d case") - - self.fluxSource = self.experimentHash.get("fluxSource", None) - self.fluxChannels = self.experimentHash.get("fluxChannels", range(8, 16)) ## wave8 - self.fluxSign = self.experimentHash.get("fluxSign", 1) - - self.ignoreEventCodeCheck = self.experimentHash.get("ignoreEventCodeCheck", None) - self.fakeBeamCode = True if self.ignoreEventCodeCheck is not None else False - - def setupROIFiles(self): - self.ROIfileNames = None - try: - self.ROIfileNames = self.experimentHash.get("ROIs", None) - self.ROIs = [] - for f in self.ROIfileNames: - self.ROIs.append(np.load(f)) - self.ROI = self.ROIs[0] if len(self.ROIs) >= 1 else None - except Exception: - if self.ROIfileNames is not None: - print("had trouble finding" + str(self.ROIfileNames)) - logger.error("had trouble finding" + str(self.ROIfileNames)) - for currName in self.ROIfileNames: - print("had trouble finding" + currName) - logger.exception("had trouble finding" + currName) - - def setupFromCmdlineArgs(self): - self.special = self.args.special - - if not self.fakeBeamCode: ## defined in ignoreEventCodeCheck - if self.special is not None: - self.fakeBeamCode = "fakeBeamCode" in self.special - - print( - "ignoring event code check, faking beam code:" - + str(self.ignoreEventCodeCheck) - + " " - + str(self.fakeBeamCode) - ) - logger.info( - "ignoring event code check, faking beam code:" - + str(self.ignoreEventCodeCheck) - + " " - + str(self.fakeBeamCode) - ) - - ## for standalone analysis - self.file = self.args.files - self.label = "" if self.args.label is None else self.args.label - - ## analyzing xtcs - self.run = self.args.run - - self.camera = 0 if self.args.camera is None else self.args.camera - - # this is set in the config-file, but take the cmd-line value instead if it is set - if self.args.exp is not None: - self.exp = self.args.exp - - self.location = self.experimentHash.get("location", None) - if self.args.location is not None: - self.location = self.args.location - - self.maxNevents = self.args.maxNevents - self.skipNevents = self.args.skipNevents - - self.detObj = self.args.detObj - - self.threshold = None - if self.args.threshold is not None: - try: - self.threshold = eval(self.args.threshold) - except Exception as e: - print("Error evaluating threshold: " + str(e)) - logger.exception("Error evaluating threshold: " + str(e)) - self.threshold = None - - self.seedCut = None - if self.args.seedCut is not None: - try: - self.seedCut = eval(self.args.seedCut) - except Exception as e: - print("Error evaluating seedcut: " + str(e)) - logger.exception("Error evaluating seedcut: " + str(e)) - self.seedCut = None - - self.fluxCutMin = self.args.fluxCutMin - self.fluxCutMax = self.args.fluxCutMax - - self.runRange = None - if self.args.runRange is not None: - try: - self.runRange = eval(self.args.runRange) ## in case needed - except Exception as e: - print("Error evaluating runRange: " + str(e)) - logger.exception("Error evaluating runRange: " + str(e)) - self.runRange = None - - self.loadPedestalGainOffsetFiles() - - if self.args.detType == "": - ## assume epix10k for now - if self.args.nModules is not None: - self.detectorInfo.setNModules(self.args.nModules) - self.detType = self.detectorInfo.getCameraType() - else: - self.detType = self.args.detType - - self.analyzedModules = self.experimentHash.get("analyzedModules", None) - if self.analyzedModules is not None: - try: - self.analyzedModules = range(self.detectorInfo.nModules) - except Exception as e: - print("Error evaluating range: " + str(e)) - logger.info("Error evaluating range: " + str(e)) - - self.g0cut = self.detectorInfo.g0cut - if self.g0cut is not None: - self.gainBitsMask = self.g0cut - 1 - else: - self.gainBitsMask = 0xFFFF ## might be dumb. for non-autoranging - - self.negativeGain = self.detectorInfo.negativeGain ## could just use the detector info in places it's defined - - def loadPedestalGainOffsetFiles(self): - self.fivePedestalRun = self.args.fivePedestalRun ## in case needed - - self.fakePedestal = None - self.fakePedestalFile = self.args.fakePedestalFile - if self.fakePedestalFile is not None: - try: - self.fakePedestal = np.load(self.fakePedestalFile) ##cast to uint32??? - except Exception as e: - print("Error loading fake pedistal: " + str(e)) - logger.exception("Error loading fake pedistal: " + str(e)) - - self.g0PedFile = self.args.g0PedFile - if self.g0PedFile is not None: - ##self.g0Ped = np.load(self.g0PedFile) - self.g0Ped = np.array([np.load(self.g0PedFile)]) ##temp hack - print(self.g0Ped.shape) - - self.g1PedFile = self.args.g0PedFile - if self.g1PedFile is not None: - ##self.g1Ped = np.load(self.g1PedFile) - self.g1Ped = np.array([np.load(self.g1PedFile)]) ##temp hack - - self.g0GainFile = self.args.g0GainFile - if self.g0GainFile is not None: - self.g0Gain = np.load(self.g0GainFile) - - self.g1GainFile = self.args.g1GainFile - if self.g1GainFile is not None: - self.g1Gain = np.load(self.g1GainFile) - - self.offsetFile = self.args.offsetFile - if self.offsetFile is not None: - self.offset = np.load(self.offsetFile) - - def setupOutputDirString(self, analysisType): - # setup output-dir for dumping output .npy, .h5, and .png files - self.outputDir = "/%s/" % (analysisType) - if self.args.path is not None: - self.outputDir = self.args.path - # if set, output folders will be relative to OUTPUT_ROOT - # if not, they will be relative to the current script file - self.outputDir = os.getenv("OUTPUT_ROOT", ".") + self.outputDir - # check if outputDir exists, if does not create it and tell user - if not os.path.exists(self.outputDir): - print("could not find output dir: " + self.outputDir) - logger.info("could not find output dir: " + self.outputDir) - print("please create this dir, exiting...") - logger.info("please create this dir, exiting...") - exit(1) - # the following doesnt work with mpi parallelism (other thread could make dir b4 curr thread) - # print("so creating dir: " + self.outputDir) - # logger.info("creating dir: " + self.outputDir) - # os.makedirs(self.outputDir) - # give dir read, write, execute permissions - # os.chmod(self.outputDir, 0o777) - else: - print("output dir: " + self.outputDir) - logger.info("output dir: " + self.outputDir) - def setROI(self, roiFile=None, roi=None): """Call with both file name and roi to save roi to file and use, just name to load, diff --git a/calibrationSuite/psana1Base.py b/calibrationSuite/psana1Base.py index af286ef..9fec947 100755 --- a/calibrationSuite/psana1Base.py +++ b/calibrationSuite/psana1Base.py @@ -16,11 +16,12 @@ import psana from calibrationSuite.argumentParser import ArgumentParser +from calibrationSuite.psanaCommon import PsanaCommon logger = logging.getLogger(__name__) -class PsanaBase(object): +class PsanaBase(PsanaCommon): def __init__(self, analysisType="scan"): commandUsed = sys.executable + " " + " ".join(sys.argv) logger.info("Ran with cmd: " + commandUsed) @@ -52,8 +53,6 @@ def __init__(self, analysisType="scan"): print("type %s not in known types" % (self.experimentHash["detectorType"]), knownTypes) return -1 - ## self.setupPsana() - def importConfigFile(self, file_path): if not os.path.exists(file_path): print(f"The file '{file_path}' does not exist") @@ -89,17 +88,6 @@ def setupPsana(self): except Exception: self.controlData = None - def getFivePedestalRunInfo(self): - ## could do load_txt but would require full path so - if self.det is None: - self.setupPsana() - - evt = self.getEvt(self.fivePedestalRun) - self.fpGains = self.det.gain(evt) - self.fpPedestals = self.det.pedestals(evt) - self.fpStatus = self.det.status(evt) ## does this work? - self.fpRMS = self.det.rms(evt) ## does this work? - def getEvt(self, run=None): oldDs = self.ds if run is not None: @@ -112,35 +100,6 @@ def getEvt(self, run=None): self.ds = oldDs return evt - def getEvtFromRunsTooSmartForMyOwnGood(self): - for r in self.runRange: - self.run = r - self.ds = self.get_ds() - try: - evt = next(self.ds.events()) - yield evt - except Exception: - continue - - def getEvtFromRuns(self): - try: ## can't get yield to work - evt = next(self.ds.events()) - return evt - except StopIteration: - i = self.runRange.index(self.run) - try: - self.run = self.runRange[i + 1] - print("switching to run %d" % (self.run)) - logger.info("switching to run %d" % (self.run)) - self.ds = self.get_ds(self.run) - except Exception: - print("have run out of new runs") - logger.exception("have run out of new runs") - return None - ##print("get event from new run") - evt = next(self.ds.events()) - return evt - def getFlux(self, evt): try: fluxes = self.wave8.get(evt).peakA() @@ -163,15 +122,6 @@ def getFlux(self, evt): return None return f - def get_evrs(self): - if self.config is None: - self.get_config() - - self.evrs = [] - for key in list(self.config.keys()): - if key.type() == psana.EvrData.ConfigV7: - self.evrs.append(key.src()) - def isKicked(self, evt): try: evr = evt.get(psana.EvrData.DataV4, self.evrs[0]) @@ -193,9 +143,6 @@ def isKicked(self, evt): pass return kicked - def get_config(self): - self.config = self.ds.env().configStore() - def getStepGen(self): return self.ds.steps() @@ -212,6 +159,3 @@ def getRawData(self, evt, gainBitsMasked=True): def getCalibData(self, evt): return self.det.calib(evt) - - def getImage(self, evt, data=None): - return self.raw.image(evt, data) diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index aef8faa..03d14a2 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -8,7 +8,6 @@ ## the terms contained in the LICENSE.txt file. ############################################################################## # from psana import * -import importlib.util import logging ## for parallelism @@ -21,6 +20,7 @@ from mpi4py import MPI from calibrationSuite.argumentParser import ArgumentParser +from calibrationSuite.psanaCommon import PsanaCommon ##from PSCalib.NDArrIO import load_txt @@ -38,7 +38,7 @@ logger = logging.getLogger(__name__) -class PsanaBase(object): +class PsanaBase(PsanaCommon): def __init__(self, analysisType="scan"): commandUsed = sys.executable + " " + " ".join(sys.argv) logger.info("Ran with cmd: " + commandUsed) @@ -52,47 +52,17 @@ def __init__(self, analysisType="scan"): self.args = ArgumentParser().parse_args() logger.info("parsed cmdline args: " + str(self.args)) - # if the SUITE_CONFIG env var is set use that, otherwise if the cmd line arg is set use that - # if neither are set, use the default 'suiteConfig.py' file - defaultConfigFileName = "suiteConfig.py" - secondaryConfigFileName = defaultConfigFileName if self.args.configFile is None else self.args.configFile - # secondaryConfigFileName is returned if env var not set - configFileName = os.environ.get("SUITE_CONFIG", secondaryConfigFileName) - config = self.importConfigFile(configFileName) - if config is None: - print("\ncould not find or read config file: " + configFileName) - print("please set SUITE_CONFIG env-var or use the '-cf' cmd-line arg to specify a valid config file") - print("exiting...") - sys.exit(1) - self.experimentHash = config.experimentHash - knownTypes = ["epixhr", "epixM", "rixsCCD"] - if self.experimentHash["detectorType"] not in knownTypes: - print("type %s not in known types" % (self.experimentHash["detectorType"]), knownTypes) - return -1 - ##self.setupPsana() - - def importConfigFile(self, file_path): - if not os.path.exists(file_path): - print(f"The file '{file_path}' does not exist") - return None - spec = importlib.util.spec_from_file_location("config", file_path) - config_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(config_module) - return config_module - def get_ds(self, run=None): if run is None: run = self.run - ##tmpDir = '/sdf/data/lcls/ds/rix/rixx1005922/scratch/xtc'## temp + ##tmpDir = '/sdf/data/lcls/ds/rix/rixx1005922/scratch/xtc' ds = psana.DataSource( exp=self.exp, run=run, intg_det=self.experimentHash["detectorType"], max_events=self.maxNevents ) ##, dir=tmpDir) return ds def setupPsana(self): - ##print("have built basic script class, exp %s run %d" %(self.exp, self.run)) if self.runRange is None: - print(self.run) self.ds = self.get_ds(self.run) else: self.run = self.runRange[0] @@ -102,11 +72,10 @@ def setupPsana(self): try: self.step_value = self.myrun.Detector("step_value") self.step_docstring = self.myrun.Detector("step_docstring") - ##print('foo', self.step_value, self.step_docstring) except Exception: self.step_value = self.step_docstring = None - ## self.det = Detector('%s.0:%s.%d' %(self.location, self.detType, self.camera), self.ds.env()) + ## self.det = Detector('%s.0:%s.%d' %(self.location, self.detType, self.camera), self.ds.env()) ## make this less dumb to accomodate epixM etc. ## use a dict etc. self.det = self.myrun.Detector(self.experimentHash["detectorType"]) @@ -143,19 +112,6 @@ def setupPsana(self): except Exception: self.controlData = None - ## if self.mfxDg1 is None: - - def getFivePedestalRunInfo(self): - ## could do load_txt but would require full path so - if self.det is None: - self.setupPsana() - - evt = self.getEvt(self.fivePedestalRun) - self.fpGains = self.det.gain(evt) - self.fpPedestals = self.det.pedestals(evt) - self.fpStatus = self.det.status(evt) ## does this work? - self.fpRMS = self.det.rms(evt) ## does this work? - def getEvtOld(self, run=None): oldDs = self.ds if run is not None: @@ -198,35 +154,6 @@ def matchedDetEvt(self): else: continue - def getEvtFromRunsTooSmartForMyOwnGood(self): - for r in self.runRange: - self.run = r - self.ds = self.get_ds() - try: - evt = next(self.ds.events()) - yield evt - except Exception: - continue - - def getEvtFromRuns(self): - try: ## can't get yield to work - evt = next(self.ds.events()) - return evt - except StopIteration: - i = self.runRange.index(self.run) - try: - self.run = self.runRange[i + 1] - print("switching to run %d" % (self.run)) - logger.info("switching to run %d" % (self.run)) - self.ds = self.get_ds(self.run) - except Exception: - print("have run out of new runs") - logger.exception("have run out of new runs") - return None - ##print("get event from new run") - evt = next(self.ds.events()) - return evt - def getAllFluxes(self, evt): if evt is None: return None @@ -262,15 +189,6 @@ def getFlux(self, evt): ##return 1 return self.flux - def get_evrs(self): - if self.config is None: - self.get_config() - - self.evrs = [] - for key in list(self.config.keys()): - if key.type() == psana.EvrData.ConfigV7: - self.evrs.append(key.src()) - def getEventCodes(self, evt): return self.timing.raw.eventcodes(evt) @@ -282,9 +200,6 @@ def isKicked(self, evt): ##print(allcodes) return allcodes[self.desiredCodes["120Hz"]] - def get_config(self): - self.config = self.ds.env().configStore() - def getStepGen(self): return self.myrun.steps() @@ -323,9 +238,6 @@ def getCalibData(self, evt): frames = self.det.raw.calib(evt) return frames - def getImage(self, evt, data=None): - return self.det.raw.image(evt, data) - def getTimestamp(self, evt): return evt.timestamp diff --git a/calibrationSuite/psanaCommon.py b/calibrationSuite/psanaCommon.py new file mode 100755 index 0000000..3482008 --- /dev/null +++ b/calibrationSuite/psanaCommon.py @@ -0,0 +1,328 @@ +############################################################################## +## This file is part of 'SLAC Beamtime Calibration Suite'. +## It is subject to the license terms in the LICENSE.txt file found in the +## top-level directory of this distribution and at: +## https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +## No part of 'SLAC Beamtime Calibration Suite', including this file, +## may be copied, modified, propagated, or distributed except according to +## the terms contained in the LICENSE.txt file. +############################################################################## +import psana +import logging +import importlib.util + +## for parallelism +import os +import sys + +from calibrationSuite.argumentParser import ArgumentParser +from calibrationSuite.detectorInfo import DetectorInfo + +logger = logging.getLogger(__name__) + + +class PsanaCommon(object): + def __init__(self, analysisType="scan"): + print("in psanaCommon") + logger.info("in psanaCommon") + + self.loadExperimentHashFromConfig() + self.setupFromExperimentHash() + self.setupFromCmdlineArgs() + self.setupOutputDirString(analysisType) + + def loadExperimentHashFromConfig(self): + # if the SUITE_CONFIG env var is set use that, otherwise if the cmd line arg is set use that + # if neither are set, use the default 'suiteConfig.py' file + defaultConfigFileName = "suiteConfig.py" + secondaryConfigFileName = defaultConfigFileName if self.args.configFile is None else self.args.configFile + # secondaryConfigFileName is returned if env var not set + configFileName = os.environ.get("SUITE_CONFIG", secondaryConfigFileName) + config = self.importConfigFile(configFileName) + if config is None: + print("\ncould not find or read config file: " + configFileName) + print("please set SUITE_CONFIG env-var or use the '-cf' cmd-line arg to specify a valid config file") + print("exiting...") + sys.exit(1) + self.experimentHash = config.experimentHash + knownTypes = ["epixhr", "epixM", "rixsCCD"] + if self.experimentHash["detectorType"] not in knownTypes: + print("type %s not in known types" % (self.experimentHash["detectorType"]), knownTypes) + return -1 + + def importConfigFile(self, file_path): + if not os.path.exists(file_path): + print(f"The file '{file_path}' does not exist") + return None + spec = importlib.util.spec_from_file_location("config", file_path) + config_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(config_module) + return config_module + + def setupFromExperimentHash(self): + try: + self.detectorInfo = DetectorInfo( + self.experimentHash["detectorType"], self.experimentHash["detectorSubtype"] + ) + except Exception: + self.detectorInfo = DetectorInfo(self.experimentHash["detectorType"]) + + self.exp = self.experimentHash.get("exp", None) + + self.setupROIFiles() + + self.singlePixels = self.experimentHash.get("singlePixels", None) + + self.regionSlice = self.experimentHash.get("regionSlice", None) + if self.regionSlice is not None: + ## n.b. assumes 3d slice now + self.sliceCoordinates = [ + [self.regionSlice[1].start, self.regionSlice[1].stop], + [self.regionSlice[2].start, self.regionSlice[2].stop], + ] + sc = self.sliceCoordinates + self.sliceEdges = [sc[0][1] - sc[0][0], sc[1][1] - sc[1][0]] + + ## handle 1d rixs ccd data + if self.detectorInfo.dimension == 2: + self.regionSlice = self.regionSlice[0], self.regionSlice[2] + print("remapping regionSlice to handle 1d case") + logger.info("remapping regionSlice to handle 1d case") + + self.fluxSource = self.experimentHash.get("fluxSource", None) + self.fluxChannels = self.experimentHash.get("fluxChannels", range(8, 16)) ## wave8 + self.fluxSign = self.experimentHash.get("fluxSign", 1) + + self.ignoreEventCodeCheck = self.experimentHash.get("ignoreEventCodeCheck", None) + self.fakeBeamCode = True if self.ignoreEventCodeCheck is not None else False + + def setupROIFiles(self): + self.ROIfileNames = None + try: + self.ROIfileNames = self.experimentHash.get("ROIs", None) + self.ROIs = [] + for f in self.ROIfileNames: + self.ROIs.append(np.load(f)) + self.ROI = self.ROIs[0] if len(self.ROIs) >= 1 else None + except Exception: + if self.ROIfileNames is not None: + print("had trouble finding" + str(self.ROIfileNames)) + logger.error("had trouble finding" + str(self.ROIfileNames)) + for currName in self.ROIfileNames: + print("had trouble finding" + currName) + logger.exception("had trouble finding" + currName) + + def setupFromCmdlineArgs(self): + self.special = self.args.special + + if not self.fakeBeamCode: ## defined in ignoreEventCodeCheck + if self.special is not None: + self.fakeBeamCode = "fakeBeamCode" in self.special + + print( + "ignoring event code check, faking beam code:" + + str(self.ignoreEventCodeCheck) + + " " + + str(self.fakeBeamCode) + ) + logger.info( + "ignoring event code check, faking beam code:" + + str(self.ignoreEventCodeCheck) + + " " + + str(self.fakeBeamCode) + ) + + ## for standalone analysis + self.file = self.args.files + self.label = "" if self.args.label is None else self.args.label + + ## analyzing xtcs + self.run = self.args.run + + self.camera = 0 if self.args.camera is None else self.args.camera + + # this is set in the config-file, but take the cmd-line value instead if it is set + if self.args.exp is not None: + self.exp = self.args.exp + + self.location = self.experimentHash.get("location", None) + if self.args.location is not None: + self.location = self.args.location + + self.maxNevents = self.args.maxNevents + self.skipNevents = self.args.skipNevents + + self.detObj = self.args.detObj + + self.threshold = None + if self.args.threshold is not None: + try: + self.threshold = eval(self.args.threshold) + except Exception as e: + print("Error evaluating threshold: " + str(e)) + logger.exception("Error evaluating threshold: " + str(e)) + self.threshold = None + + self.seedCut = None + if self.args.seedCut is not None: + try: + self.seedCut = eval(self.args.seedCut) + except Exception as e: + print("Error evaluating seedcut: " + str(e)) + logger.exception("Error evaluating seedcut: " + str(e)) + self.seedCut = None + + self.fluxCutMin = self.args.fluxCutMin + self.fluxCutMax = self.args.fluxCutMax + + self.runRange = None + if self.args.runRange is not None: + try: + self.runRange = eval(self.args.runRange) ## in case needed + except Exception as e: + print("Error evaluating runRange: " + str(e)) + logger.exception("Error evaluating runRange: " + str(e)) + self.runRange = None + + self.loadPedestalGainOffsetFiles() + + if self.args.detType == "": + ## assume epix10k for now + if self.args.nModules is not None: + self.detectorInfo.setNModules(self.args.nModules) + self.detType = self.detectorInfo.getCameraType() + else: + self.detType = self.args.detType + + self.analyzedModules = self.experimentHash.get("analyzedModules", None) + if self.analyzedModules is not None: + try: + self.analyzedModules = range(self.detectorInfo.nModules) + except Exception as e: + print("Error evaluating range: " + str(e)) + logger.info("Error evaluating range: " + str(e)) + + self.g0cut = self.detectorInfo.g0cut + if self.g0cut is not None: + self.gainBitsMask = self.g0cut - 1 + else: + self.gainBitsMask = 0xFFFF ## might be dumb. for non-autoranging + + self.negativeGain = self.detectorInfo.negativeGain ## could just use the detector info in places it's defined + + def loadPedestalGainOffsetFiles(self): + self.fivePedestalRun = self.args.fivePedestalRun ## in case needed + + self.fakePedestal = None + self.fakePedestalFile = self.args.fakePedestalFile + if self.fakePedestalFile is not None: + try: + self.fakePedestal = np.load(self.fakePedestalFile) ##cast to uint32??? + except Exception as e: + print("Error loading fake pedistal: " + str(e)) + logger.exception("Error loading fake pedistal: " + str(e)) + + self.g0PedFile = self.args.g0PedFile + if self.g0PedFile is not None: + ##self.g0Ped = np.load(self.g0PedFile) + self.g0Ped = np.array([np.load(self.g0PedFile)]) ##temp hack + print(self.g0Ped.shape) + + self.g1PedFile = self.args.g0PedFile + if self.g1PedFile is not None: + ##self.g1Ped = np.load(self.g1PedFile) + self.g1Ped = np.array([np.load(self.g1PedFile)]) ##temp hack + + self.g0GainFile = self.args.g0GainFile + if self.g0GainFile is not None: + self.g0Gain = np.load(self.g0GainFile) + + self.g1GainFile = self.args.g1GainFile + if self.g1GainFile is not None: + self.g1Gain = np.load(self.g1GainFile) + + self.offsetFile = self.args.offsetFile + if self.offsetFile is not None: + self.offset = np.load(self.offsetFile) + + def setupOutputDirString(self, analysisType): + # setup output-dir for dumping output .npy, .h5, and .png files + self.outputDir = "/%s/" % (analysisType) + if self.args.path is not None: + self.outputDir = self.args.path + # if set, output folders will be relative to OUTPUT_ROOT + # if not, they will be relative to the current script file + self.outputDir = os.getenv("OUTPUT_ROOT", ".") + self.outputDir + # check if outputDir exists, if does not create it and tell user + if not os.path.exists(self.outputDir): + print("could not find output dir: " + self.outputDir) + logger.info("could not find output dir: " + self.outputDir) + print("please create this dir, exiting...") + logger.info("please create this dir, exiting...") + exit(1) + # the following doesnt work with mpi parallelism (other thread could make dir b4 curr thread) + # print("so creating dir: " + self.outputDir) + # logger.info("creating dir: " + self.outputDir) + # os.makedirs(self.outputDir) + # give dir read, write, execute permissions + # os.chmod(self.outputDir, 0o777) + else: + print("output dir: " + self.outputDir) + logger.info("output dir: " + self.outputDir) + + + + def get_evrs(self): + if self.config is None: + self.get_config() + + self.evrs = [] + for key in list(self.config.keys()): + if key.type() == psana.EvrData.ConfigV7: + self.evrs.append(key.src()) + + def get_config(self): + self.config = self.ds.env().configStore() + + def getImage(self, evt, data=None): + return self.raw.image(evt, data) + + def getFivePedestalRunInfo(self): + ## could do load_txt but would require full path so + if self.det is None: + self.setupPsana() + + evt = self.getEvt(self.fivePedestalRun) + self.fpGains = self.det.gain(evt) + self.fpPedestals = self.det.pedestals(evt) + self.fpStatus = self.det.status(evt) ## does this work? + self.fpRMS = self.det.rms(evt) ## does this work? + + def getEvtFromRunsTooSmartForMyOwnGood(self): + for r in self.runRange: + self.run = r + self.ds = self.get_ds() + try: + evt = next(self.ds.events()) + yield evt + except Exception: + continue + + def getEvtFromRuns(self): + try: ## can't get yield to work + evt = next(self.ds.events()) + return evt + except StopIteration: + i = self.runRange.index(self.run) + try: + self.run = self.runRange[i + 1] + print("switching to run %d" % (self.run)) + logger.info("switching to run %d" % (self.run)) + self.ds = self.get_ds(self.run) + except Exception: + print("have run out of new runs") + logger.exception("have run out of new runs") + return None + ##print("get event from new run") + evt = next(self.ds.events()) + return evt \ No newline at end of file From 27b797113515001bfd110a51ba01e2f7a67af692 Mon Sep 17 00:00:00 2001 From: nstelter Date: Wed, 19 Jun 2024 19:19:34 -0700 Subject: [PATCH 08/16] WIP: more refactoring lib code --- calibrationSuite/psana1Base.py | 2 ++ calibrationSuite/psana2Base.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/calibrationSuite/psana1Base.py b/calibrationSuite/psana1Base.py index 9fec947..5c63337 100755 --- a/calibrationSuite/psana1Base.py +++ b/calibrationSuite/psana1Base.py @@ -23,6 +23,8 @@ class PsanaBase(PsanaCommon): def __init__(self, analysisType="scan"): + super().__init__() + commandUsed = sys.executable + " " + " ".join(sys.argv) logger.info("Ran with cmd: " + commandUsed) diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index 03d14a2..c9a3af7 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -40,6 +40,8 @@ class PsanaBase(PsanaCommon): def __init__(self, analysisType="scan"): + super().__init__() + commandUsed = sys.executable + " " + " ".join(sys.argv) logger.info("Ran with cmd: " + commandUsed) From 8a3db5a5d03eee3eebefb5e7fc6a3c510f703b5d Mon Sep 17 00:00:00 2001 From: nstelter Date: Wed, 19 Jun 2024 19:21:06 -0700 Subject: [PATCH 09/16] WIP: more refactoring lib code --- calibrationSuite/psana1Base.py | 3 --- calibrationSuite/psana2Base.py | 3 --- calibrationSuite/psanaCommon.py | 3 +++ 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/calibrationSuite/psana1Base.py b/calibrationSuite/psana1Base.py index 5c63337..765a998 100755 --- a/calibrationSuite/psana1Base.py +++ b/calibrationSuite/psana1Base.py @@ -34,9 +34,6 @@ def __init__(self, analysisType="scan"): self.g0cut = 1 << 14 self.gainBitsMask = self.g0cut - 1 - self.args = ArgumentParser().parse_args() - logger.info("parsed cmdline args: " + str(self.args)) - # if the SUITE_CONFIG env var is set use that, otherwise if the cmd line arg is set use that # if neither are set, use the default 'suiteConfig.py' file defaultConfigFileName = "suiteConfig.py" diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index c9a3af7..f4e9999 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -51,9 +51,6 @@ def __init__(self, analysisType="scan"): self.allowed_timestamp_mismatch = 1000 - self.args = ArgumentParser().parse_args() - logger.info("parsed cmdline args: " + str(self.args)) - def get_ds(self, run=None): if run is None: run = self.run diff --git a/calibrationSuite/psanaCommon.py b/calibrationSuite/psanaCommon.py index 3482008..00a1b6f 100755 --- a/calibrationSuite/psanaCommon.py +++ b/calibrationSuite/psanaCommon.py @@ -26,6 +26,9 @@ def __init__(self, analysisType="scan"): print("in psanaCommon") logger.info("in psanaCommon") + self.args = ArgumentParser().parse_args() + logger.info("parsed cmdline args: " + str(self.args)) + self.loadExperimentHashFromConfig() self.setupFromExperimentHash() self.setupFromCmdlineArgs() From 65bc8e8d3783e774a208820f58bab711a45cd701 Mon Sep 17 00:00:00 2001 From: nstelter Date: Wed, 19 Jun 2024 20:07:54 -0700 Subject: [PATCH 10/16] WIP: more refactoring lib code --- calibrationSuite/basicSuiteScript.py | 148 +---------------- calibrationSuite/psana1Base.py | 27 --- calibrationSuite/psana2Base.py | 4 +- calibrationSuite/psanaCommon.py | 236 ++++++++++++++++++++++++++- 4 files changed, 233 insertions(+), 182 deletions(-) diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index 0ef7fae..6e1f188 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -47,150 +47,4 @@ def __init__(self, analysisType="scan"): self.runCode = 280 self.daqCode = 281 self.beamCode = 283 ## per Matt - ##self.beamCode = 281 ## don't see 283... - - def setROI(self, roiFile=None, roi=None): - """Call with both file name and roi to save roi to file and use, - just name to load, - just roi to set for current job""" - if roiFile is not None: - if roi is None: - self.ROIfile = roiFile - self.ROI = np.load(roiFile) - return - else: - np.save(roiFile, roi) - self.ROI = roi - - def sliceToDetector(self, sliceRow, sliceCol): ## cp from AnalyzeH5: import? - return sliceRow + self.sliceCoordinates[0][0], sliceCol + self.sliceCoordinates[1][0] - - def noCommonModeCorrection(self, frames): - return frames - - def regionCommonModeCorrection(self, frame, region, arbitraryCut=1000): - ## this takes a 2d frame - ## cut keeps photons in common mode - e.g. set to <<1 photon - - regionCM = np.median(frame[region][frame[region] < arbitraryCut]) - return frame - regionCM - - def rowCommonModeCorrection3d(self, frames, arbitraryCut=1000): - for module in self.analyzedModules: - frames[module] = self.rowCommonModeCorrection(frames[module], arbitraryCut) - return frames - - def colCommonModeCorrection3d(self, frames, arbitraryCut=1000): - for module in self.analyzedModules: - frames[module] = self.colCommonModeCorrection(frames[module], arbitraryCut) - return frames - - def rowCommonModeCorrection(self, frame, arbitraryCut=1000): - ## this takes a 2d object - ## cut keeps photons in common mode - e.g. set to <<1 photon - - for r in range(self.detectorInfo.nRows): - colOffset = 0 - for b in range(0, self.detectorInfo.nBanksCol): - ##for b in range(0, 2): - try: - rowCM = np.median( - frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank][ - frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank] < arbitraryCut - ] - ) - frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank] -= rowCM - except Exception: - rowCM = -666 - print("rowCM problem") - logger.error("rowCM problem") - print(frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank]) - colOffset += self.detectorInfo.nColsPerBank - return frame - - def colCommonModeCorrection(self, frame, arbitraryCut=1000): - ## this takes a 2d frame - ## cut keeps photons in common mode - e.g. set to <<1 photon - - ##rand = np.random.random() - for c in range(self.detectorInfo.nCols): - rowOffset = 0 - for b in range(0, self.detectorInfo.nBanksRow): - try: - colCM = np.median( - frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c][ - frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c] < arbitraryCut - ] - ) - frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c] -= colCM - except Exception: - colCM = -666 - print("colCM problem") - logger.error("colCM problem") - print(frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank], c) - rowOffset += self.detectorInfo.nRowsPerBank - return frame - - def isBeamEvent(self, evt): - if self.ignoreEventCodeCheck: - return True - ec = self.getEventCodes(evt) - ##print(ec[280], ec[281], ec[282], ec[283], ec[284], ec[285] ) - if ec[self.runCode]: - self.nRunCodeEvents += 1 - if ec[self.daqCode]: - self.nDaqCodeEvents += 1 - if self.fakeBeamCode: - return True - if ec[self.beamCode]: - self.nBeamCodeEvents += 1 - return True - ## for FEE, ASC, ... - return self.fakeBeamCode ##False - - def dumpEventCodeStatistics(self): - print( - "have counted %d run triggers, %d DAQ triggers, %d beam events" - % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) - ) - logger.info( - "have counted %d run triggers, %d DAQ triggers, %d beam events" - % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) - ) - - def getRawData(self, evt, gainBitsMasked=True, negativeGain=False): - frames = self.plainGetRawData(evt) - if frames is None: - return None - if False and self.special: ## turned off for a tiny bit of speed - if "thirteenBits" in self.special: - frames = frames & 0xFFFE - ##print("13bits") - elif "twelveBits" in self.special: - frames = frames & 0xFFFC - ##print("12bits") - elif "elevenBits" in self.special: - frames = frames & 0xFFF8 - ##print("11bits") - elif "tenBits" in self.special: - frames = frames & 0xFFF0 - ##print("10bits") - if self.negativeGain or negativeGain: - zeroPixels = frames == 0 - maskedData = frames & self.gainBitsMask - gainData = frames - maskedData - frames = gainData + self.gainBitsMask - maskedData - frames[zeroPixels] = 0 - if gainBitsMasked: - return frames & self.gainBitsMask - return frames - - def addFakePhotons(self, frames, occupancy, E, width): - shape = frames.shape - occ = np.random.random(shape) - fakes = np.random.normal(E, width, shape) - fakes[occ > occupancy] = 0 - return frames + fakes, (fakes > 0).sum() - - def getNswitchedPixels(self, data, region=None): - return ((data >= self.g0cut) * 1).sum() + ##self.beamCode = 281 ## don't see 283... \ No newline at end of file diff --git a/calibrationSuite/psana1Base.py b/calibrationSuite/psana1Base.py index 765a998..3895d54 100755 --- a/calibrationSuite/psana1Base.py +++ b/calibrationSuite/psana1Base.py @@ -34,33 +34,6 @@ def __init__(self, analysisType="scan"): self.g0cut = 1 << 14 self.gainBitsMask = self.g0cut - 1 - # if the SUITE_CONFIG env var is set use that, otherwise if the cmd line arg is set use that - # if neither are set, use the default 'suiteConfig.py' file - defaultConfigFileName = "suiteConfig.py" - secondaryConfigFileName = defaultConfigFileName if self.args.configFile is None else self.args.configFile - # secondaryConfigFileName is returned if env var not set - configFileName = os.environ.get("SUITE_CONFIG", secondaryConfigFileName) - config = self.importConfigFile(configFileName) - if config is None: - print("\ncould not find or read config file: " + configFileName) - print("please set SUITE_CONFIG env-var or use the '-cf' cmd-line arg to specify a valid config file") - print("exiting...") - sys.exit(1) - self.experimentHash = config.experimentHash - knownTypes = ["epixhr", "epixM", "rixsCCD"] - if self.experimentHash["detectorType"] not in knownTypes: - print("type %s not in known types" % (self.experimentHash["detectorType"]), knownTypes) - return -1 - - def importConfigFile(self, file_path): - if not os.path.exists(file_path): - print(f"The file '{file_path}' does not exist") - return None - spec = importlib.util.spec_from_file_location("config", file_path) - config_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(config_module) - return config_module - def get_ds(self, run=None): if run is None: run = self.run diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index f4e9999..885e60c 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -165,7 +165,7 @@ def _getFlux(self, evt): if self.mfxDg1 is None: return None - ## f = self.mfxDg1.raw.peakAmplitude(evt)[self.fluxChannels].mean()*self.fluxSign + ##f = self.mfxDg1.raw.peakAmplitude(evt)[self.fluxChannels].mean()*self.fluxSign try: f = self.mfxDg1.raw.peakAmplitude(evt)[self.fluxChannels].mean() * self.fluxSign ##print(f) @@ -236,7 +236,7 @@ def plainGetRawData(self, evt): def getCalibData(self, evt): frames = self.det.raw.calib(evt) return frames - + def getTimestamp(self, evt): return evt.timestamp diff --git a/calibrationSuite/psanaCommon.py b/calibrationSuite/psanaCommon.py index 00a1b6f..0881528 100755 --- a/calibrationSuite/psanaCommon.py +++ b/calibrationSuite/psanaCommon.py @@ -7,6 +7,12 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## + +""" +This class has setup-code and variables, along with some utility functions, common for both psana bases. +(psana1Base and psana2Base must both inherit from PsanaCommon!) +""" + import psana import logging import importlib.util @@ -14,6 +20,7 @@ ## for parallelism import os import sys +import np from calibrationSuite.argumentParser import ArgumentParser from calibrationSuite.detectorInfo import DetectorInfo @@ -28,13 +35,24 @@ def __init__(self, analysisType="scan"): self.args = ArgumentParser().parse_args() logger.info("parsed cmdline args: " + str(self.args)) - + + # Variable-setup calls self.loadExperimentHashFromConfig() self.setupFromExperimentHash() self.setupFromCmdlineArgs() self.setupOutputDirString(analysisType) + #### Start of setup related functions #### + def loadExperimentHashFromConfig(self): + """ + Loads the experiment config file (ex: beamtime-calibration-suite/config_files/epixMSuiteConfig.py) + The config file is determined by checking the SUITE_CONFIG environment variable, + then a command-line argument, and finally a default file 'suiteConfig.py' (in this order of precedence). + If file cannot be found or read, exit the program. + The function also validates the detector type against known types (with info from the config). + """ + # if the SUITE_CONFIG env var is set use that, otherwise if the cmd line arg is set use that # if neither are set, use the default 'suiteConfig.py' file defaultConfigFileName = "suiteConfig.py" @@ -46,23 +64,44 @@ def loadExperimentHashFromConfig(self): print("\ncould not find or read config file: " + configFileName) print("please set SUITE_CONFIG env-var or use the '-cf' cmd-line arg to specify a valid config file") print("exiting...") + logger.error("\ncould not find or read config file: " + configFileName) + logger.error("please set SUITE_CONFIG env-var or use the '-cf' cmd-line arg to specify a valid config file") + logger.error("exiting...") sys.exit(1) self.experimentHash = config.experimentHash knownTypes = ["epixhr", "epixM", "rixsCCD"] if self.experimentHash["detectorType"] not in knownTypes: print("type %s not in known types" % (self.experimentHash["detectorType"]), knownTypes) - return -1 + print("exiting...") + logger.error("type %s not in known types" % (self.experimentHash["detectorType"]), knownTypes) + logger.error("exiting...") + sys.exit(1) def importConfigFile(self, file_path): + """ + Imports config file as a Python module. See loadExperimentHashFromConfig() for how the config file + is specified by the user. + """ if not os.path.exists(file_path): - print(f"The file '{file_path}' does not exist") + print("The file " + file_path + " does not exist") + logger.error("The file " + file_path + " does not exist") return None spec = importlib.util.spec_from_file_location("config", file_path) config_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(config_module) + try: + spec.loader.exec_module(config_module) + except Exception as e: + print("Error executing config-file code: " + str(e)) + logger.info("Error executing config-file code: " + str(e)) + return None return config_module def setupFromExperimentHash(self): + """ + Sets up configurations and parameters based on user-specified config file. + Initializes necessary attributes and handles exceptions on specific evaluations. + """ + try: self.detectorInfo = DetectorInfo( self.experimentHash["detectorType"], self.experimentHash["detectorSubtype"] @@ -100,6 +139,12 @@ def setupFromExperimentHash(self): self.fakeBeamCode = True if self.ignoreEventCodeCheck is not None else False def setupROIFiles(self): + """ + Sets up the ROI (Region of Interest) files. + Loads ROI data from the file names specified in the config file. + If any errors occur during the process, logs and prints appropriate error messages. + """ + self.ROIfileNames = None try: self.ROIfileNames = self.experimentHash.get("ROIs", None) @@ -116,6 +161,11 @@ def setupROIFiles(self): logger.exception("had trouble finding" + currName) def setupFromCmdlineArgs(self): + """ + Sets up configurations and parameters based on user-entered command-line arguments. + Initializes necessary attributes and handles exceptions on specific evaluations. + """ + self.special = self.args.special if not self.fakeBeamCode: ## defined in ignoreEventCodeCheck @@ -214,6 +264,11 @@ def setupFromCmdlineArgs(self): self.negativeGain = self.detectorInfo.negativeGain ## could just use the detector info in places it's defined def loadPedestalGainOffsetFiles(self): + """ + Loads the pedestal, gain, and offset np files for analysis. + The files-locations are specified with cmd-line arguments. + """ + self.fivePedestalRun = self.args.fivePedestalRun ## in case needed self.fakePedestal = None @@ -249,7 +304,13 @@ def loadPedestalGainOffsetFiles(self): self.offset = np.load(self.offsetFile) def setupOutputDirString(self, analysisType): - # setup output-dir for dumping output .npy, .h5, and .png files + """ + Sets up the output directory for saving output file (default dir-name is based on the analysis type). + The output directory can be specified via a command-line argument, and its location must be specified relative to the + 'OUTPUT_ROOT environment variable. If the directory does not exist, it logs error and exits. + """ + + # output dir is where we dump .npy, .h5, and .png files self.outputDir = "/%s/" % (analysisType) if self.args.path is not None: self.outputDir = self.args.path @@ -273,7 +334,9 @@ def setupOutputDirString(self, analysisType): print("output dir: " + self.outputDir) logger.info("output dir: " + self.outputDir) + #### End of setup related functions #### + #### Start of common getter functions #### def get_evrs(self): if self.config is None: @@ -328,4 +391,165 @@ def getEvtFromRuns(self): return None ##print("get event from new run") evt = next(self.ds.events()) - return evt \ No newline at end of file + return evt + + def getRawData(self, evt, gainBitsMasked=True, negativeGain=False): + frames = self.plainGetRawData(evt) + if frames is None: + return None + if False and self.special: ## turned off for a tiny bit of speed + if "thirteenBits" in self.special: + frames = frames & 0xFFFE + ##print("13bits") + elif "twelveBits" in self.special: + frames = frames & 0xFFFC + ##print("12bits") + elif "elevenBits" in self.special: + frames = frames & 0xFFF8 + ##print("11bits") + elif "tenBits" in self.special: + frames = frames & 0xFFF0 + ##print("10bits") + if self.negativeGain or negativeGain: + zeroPixels = frames == 0 + maskedData = frames & self.gainBitsMask + gainData = frames - maskedData + frames = gainData + self.gainBitsMask - maskedData + frames[zeroPixels] = 0 + if gainBitsMasked: + return frames & self.gainBitsMask + return frames + + def addFakePhotons(self, frames, occupancy, E, width): + shape = frames.shape + occ = np.random.random(shape) + fakes = np.random.normal(E, width, shape) + fakes[occ > occupancy] = 0 + return frames + fakes, (fakes > 0).sum() + + #### End of common getter functions #### + + #### Start of common setter functions #### + + def setROI(self, roiFile=None, roi=None): + """Call with both file name and roi to save roi to file and use, + just name to load, + just roi to set for current job""" + if roiFile is not None: + if roi is None: + self.ROIfile = roiFile + self.ROI = np.load(roiFile) + return + else: + np.save(roiFile, roi) + self.ROI = roi + + #### End of common setter functions #### + + #### Start of common utility functions #### + + def sliceToDetector(self, sliceRow, sliceCol): ## cp from AnalyzeH5: import? + return sliceRow + self.sliceCoordinates[0][0], sliceCol + self.sliceCoordinates[1][0] + + def getNswitchedPixels(self, data, region=None): + return ((data >= self.g0cut) * 1).sum() + + def dumpEventCodeStatistics(self): + print( + "have counted %d run triggers, %d DAQ triggers, %d beam events" + % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) + ) + logger.info( + "have counted %d run triggers, %d DAQ triggers, %d beam events" + % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) + ) + + def isBeamEvent(self, evt): + if self.ignoreEventCodeCheck: + return True + ec = self.getEventCodes(evt) + ##print(ec[280], ec[281], ec[282], ec[283], ec[284], ec[285] ) + if ec[self.runCode]: + self.nRunCodeEvents += 1 + if ec[self.daqCode]: + self.nDaqCodeEvents += 1 + if self.fakeBeamCode: + return True + if ec[self.beamCode]: + self.nBeamCodeEvents += 1 + return True + ## for FEE, ASC, ... + return self.fakeBeamCode ##False + + #### End of common utility functions #### + + #### Start of analysis/correction functions #### + + def noCommonModeCorrection(self, frames): + return frames + + def regionCommonModeCorrection(self, frame, region, arbitraryCut=1000): + ## this takes a 2d frame + ## cut keeps photons in common mode - e.g. set to <<1 photon + + regionCM = np.median(frame[region][frame[region] < arbitraryCut]) + return frame - regionCM + + def rowCommonModeCorrection3d(self, frames, arbitraryCut=1000): + for module in self.analyzedModules: + frames[module] = self.rowCommonModeCorrection(frames[module], arbitraryCut) + return frames + + def colCommonModeCorrection3d(self, frames, arbitraryCut=1000): + for module in self.analyzedModules: + frames[module] = self.colCommonModeCorrection(frames[module], arbitraryCut) + return frames + + + def rowCommonModeCorrection(self, frame, arbitraryCut=1000): + ## this takes a 2d object + ## cut keeps photons in common mode - e.g. set to <<1 photon + + for r in range(self.detectorInfo.nRows): + colOffset = 0 + for b in range(0, self.detectorInfo.nBanksCol): + ##for b in range(0, 2): + try: + rowCM = np.median( + frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank][ + frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank] < arbitraryCut + ] + ) + frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank] -= rowCM + except Exception: + rowCM = -666 + print("rowCM problem") + logger.error("rowCM problem") + print(frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank]) + colOffset += self.detectorInfo.nColsPerBank + return frame + + def colCommonModeCorrection(self, frame, arbitraryCut=1000): + ## this takes a 2d frame + ## cut keeps photons in common mode - e.g. set to <<1 photon + + ##rand = np.random.random() + for c in range(self.detectorInfo.nCols): + rowOffset = 0 + for b in range(0, self.detectorInfo.nBanksRow): + try: + colCM = np.median( + frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c][ + frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c] < arbitraryCut + ] + ) + frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c] -= colCM + except Exception: + colCM = -666 + print("colCM problem") + logger.error("colCM problem") + print(frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank], c) + rowOffset += self.detectorInfo.nRowsPerBank + return frame + + #### End of analysis/correction functions #### From 1418b81cfbf2b37965362bb43153de74488f7428 Mon Sep 17 00:00:00 2001 From: nstelter Date: Wed, 19 Jun 2024 20:29:45 -0700 Subject: [PATCH 11/16] WIP: more refactoring lib code --- calibrationSuite/psana1Base.py | 5 +++-- calibrationSuite/psana2Base.py | 6 ++++-- calibrationSuite/psanaCommon.py | 2 -- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/calibrationSuite/psana1Base.py b/calibrationSuite/psana1Base.py index 3895d54..459c75e 100755 --- a/calibrationSuite/psana1Base.py +++ b/calibrationSuite/psana1Base.py @@ -14,8 +14,6 @@ import sys import psana - -from calibrationSuite.argumentParser import ArgumentParser from calibrationSuite.psanaCommon import PsanaCommon logger = logging.getLogger(__name__) @@ -131,3 +129,6 @@ def getRawData(self, evt, gainBitsMasked=True): def getCalibData(self, evt): return self.det.calib(evt) + + def getImage(self, evt, data=None): + return self.raw.image(evt, data) \ No newline at end of file diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index 885e60c..1155950 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -14,12 +14,11 @@ import os import sys -import psana ## standard from mpi4py import MPI -from calibrationSuite.argumentParser import ArgumentParser +import psana from calibrationSuite.psanaCommon import PsanaCommon ##from PSCalib.NDArrIO import load_txt @@ -237,6 +236,9 @@ def getCalibData(self, evt): frames = self.det.raw.calib(evt) return frames + def getImage(self, evt, data=None): + return self.det.raw.image(evt, data) + def getTimestamp(self, evt): return evt.timestamp diff --git a/calibrationSuite/psanaCommon.py b/calibrationSuite/psanaCommon.py index 0881528..337ed7b 100755 --- a/calibrationSuite/psanaCommon.py +++ b/calibrationSuite/psanaCommon.py @@ -350,8 +350,6 @@ def get_evrs(self): def get_config(self): self.config = self.ds.env().configStore() - def getImage(self, evt, data=None): - return self.raw.image(evt, data) def getFivePedestalRunInfo(self): ## could do load_txt but would require full path so From 9077bb543f5b090cd7617d257916e4be5fd20146 Mon Sep 17 00:00:00 2001 From: nstelter Date: Wed, 19 Jun 2024 20:31:02 -0700 Subject: [PATCH 12/16] WIP: more refactoring lib code --- calibrationSuite/psanaCommon.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/calibrationSuite/psanaCommon.py b/calibrationSuite/psanaCommon.py index 337ed7b..b942f3d 100755 --- a/calibrationSuite/psanaCommon.py +++ b/calibrationSuite/psanaCommon.py @@ -20,7 +20,7 @@ ## for parallelism import os import sys -import np +import numpy as np from calibrationSuite.argumentParser import ArgumentParser from calibrationSuite.detectorInfo import DetectorInfo @@ -50,7 +50,6 @@ def loadExperimentHashFromConfig(self): The config file is determined by checking the SUITE_CONFIG environment variable, then a command-line argument, and finally a default file 'suiteConfig.py' (in this order of precedence). If file cannot be found or read, exit the program. - The function also validates the detector type against known types (with info from the config). """ # if the SUITE_CONFIG env var is set use that, otherwise if the cmd line arg is set use that @@ -69,13 +68,6 @@ def loadExperimentHashFromConfig(self): logger.error("exiting...") sys.exit(1) self.experimentHash = config.experimentHash - knownTypes = ["epixhr", "epixM", "rixsCCD"] - if self.experimentHash["detectorType"] not in knownTypes: - print("type %s not in known types" % (self.experimentHash["detectorType"]), knownTypes) - print("exiting...") - logger.error("type %s not in known types" % (self.experimentHash["detectorType"]), knownTypes) - logger.error("exiting...") - sys.exit(1) def importConfigFile(self, file_path): """ From f0758194e21605be2e1344b8533b6f593bee6154 Mon Sep 17 00:00:00 2001 From: nstelter Date: Wed, 19 Jun 2024 20:55:46 -0700 Subject: [PATCH 13/16] WIP: more refactoring lib code --- calibrationSuite/basicSuiteScript.py | 240 ++++++++++++++++++++++-- calibrationSuite/psana1Base.py | 8 +- calibrationSuite/psana2Base.py | 8 +- calibrationSuite/psanaCommon.py | 239 ++--------------------- suite_scripts/DeltaTtest.py | 6 +- suite_scripts/EventScanParallelSlice.py | 6 +- 6 files changed, 257 insertions(+), 250 deletions(-) diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index 6e1f188..3cc0d94 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -7,6 +7,12 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## +""" +This class acts as an entry-point into the psana-based utility of the library. +(should be used in high-level scripts as follows: 'from calibrationSuite.basicSuiteScript import BasicSuiteScript') +It contains utility functions, analysis functions, and some getters and setters, that can be used with both 'psana1Base' and 'psana2Base'. +""" + import logging import os @@ -21,11 +27,6 @@ print("psana2") from calibrationSuite.psana2Base import PsanaBase - -def sortArrayByList(a, data): - return [x for _, x in sorted(zip(a, data), key=lambda pair: pair[0])] - - class BasicSuiteScript(PsanaBase): def __init__(self, analysisType="scan"): super().__init__() @@ -34,17 +35,222 @@ def __init__(self, analysisType="scan"): logger.info("in BasicSuiteScript, inheriting from PsanaBase, type is psana%d" % (self.psanaType)) self.className = self.__class__.__name__ - self.gainModes = {"FH": 0, "FM": 1, "FL": 2, "AHL-H": 3, "AML-M": 4, "AHL-L": 5, "AML-L": 6} - self.ePix10k_cameraTypes = {1: "Epix10ka", 4: "Epix10kaQuad", 16: "Epix10ka2M"} + + #### Start of common getter functions #### + + def get_evrs(self): + if self.config is None: + self.get_config() + + self.evrs = [] + for key in list(self.config.keys()): + if key.type() == psana.EvrData.ConfigV7: + self.evrs.append(key.src()) + + def get_config(self): + self.config = self.ds.env().configStore() + + + def getFivePedestalRunInfo(self): + ## could do load_txt but would require full path so + if self.det is None: + self.setupPsana() + + evt = self.getEvt(self.fivePedestalRun) + self.fpGains = self.det.gain(evt) + self.fpPedestals = self.det.pedestals(evt) + self.fpStatus = self.det.status(evt) ## does this work? + self.fpRMS = self.det.rms(evt) ## does this work? + + def getEvtFromRunsTooSmartForMyOwnGood(self): + for r in self.runRange: + self.run = r + self.ds = self.get_ds() + try: + evt = next(self.ds.events()) + yield evt + except Exception: + continue + + def getEvtFromRuns(self): + try: ## can't get yield to work + evt = next(self.ds.events()) + return evt + except StopIteration: + i = self.runRange.index(self.run) + try: + self.run = self.runRange[i + 1] + print("switching to run %d" % (self.run)) + logger.info("switching to run %d" % (self.run)) + self.ds = self.get_ds(self.run) + except Exception: + print("have run out of new runs") + logger.exception("have run out of new runs") + return None + ##print("get event from new run") + evt = next(self.ds.events()) + return evt + + def getRawData(self, evt, gainBitsMasked=True, negativeGain=False): + frames = self.plainGetRawData(evt) + if frames is None: + return None + if False and self.special: ## turned off for a tiny bit of speed + if "thirteenBits" in self.special: + frames = frames & 0xFFFE + ##print("13bits") + elif "twelveBits" in self.special: + frames = frames & 0xFFFC + ##print("12bits") + elif "elevenBits" in self.special: + frames = frames & 0xFFF8 + ##print("11bits") + elif "tenBits" in self.special: + frames = frames & 0xFFF0 + ##print("10bits") + if self.negativeGain or negativeGain: + zeroPixels = frames == 0 + maskedData = frames & self.gainBitsMask + gainData = frames - maskedData + frames = gainData + self.gainBitsMask - maskedData + frames[zeroPixels] = 0 + if gainBitsMasked: + return frames & self.gainBitsMask + return frames + + def addFakePhotons(self, frames, occupancy, E, width): + shape = frames.shape + occ = np.random.random(shape) + fakes = np.random.normal(E, width, shape) + fakes[occ > occupancy] = 0 + return frames + fakes, (fakes > 0).sum() + + #### End of common getter functions #### + + #### Start of common setter functions #### + + def setROI(self, roiFile=None, roi=None): + """Call with both file name and roi to save roi to file and use, + just name to load, + just roi to set for current job""" + if roiFile is not None: + if roi is None: + self.ROIfile = roiFile + self.ROI = np.load(roiFile) + return + else: + np.save(roiFile, roi) + self.ROI = roi + + #### End of common setter functions #### + + #### Start of common utility functions #### + + def sortArrayByList(a, data): + return [x for _, x in sorted(zip(a, data), key=lambda pair: pair[0])] + + def sliceToDetector(self, sliceRow, sliceCol): ## cp from AnalyzeH5: import? + return sliceRow + self.sliceCoordinates[0][0], sliceCol + self.sliceCoordinates[1][0] + + def getNswitchedPixels(self, data, region=None): + return ((data >= self.g0cut) * 1).sum() + + def dumpEventCodeStatistics(self): + print( + "have counted %d run triggers, %d DAQ triggers, %d beam events" + % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) + ) + logger.info( + "have counted %d run triggers, %d DAQ triggers, %d beam events" + % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) + ) + + def isBeamEvent(self, evt): + if self.ignoreEventCodeCheck: + return True + ec = self.getEventCodes(evt) + ##print(ec[280], ec[281], ec[282], ec[283], ec[284], ec[285] ) + if ec[self.runCode]: + self.nRunCodeEvents += 1 + if ec[self.daqCode]: + self.nDaqCodeEvents += 1 + if self.fakeBeamCode: + return True + if ec[self.beamCode]: + self.nBeamCodeEvents += 1 + return True + ## for FEE, ASC, ... + return self.fakeBeamCode ##False + + #### End of common utility functions #### + + #### Start of analysis/correction functions #### + + def noCommonModeCorrection(self, frames): + return frames + + def regionCommonModeCorrection(self, frame, region, arbitraryCut=1000): + ## this takes a 2d frame + ## cut keeps photons in common mode - e.g. set to <<1 photon + + regionCM = np.median(frame[region][frame[region] < arbitraryCut]) + return frame - regionCM + + def rowCommonModeCorrection3d(self, frames, arbitraryCut=1000): + for module in self.analyzedModules: + frames[module] = self.rowCommonModeCorrection(frames[module], arbitraryCut) + return frames + + def colCommonModeCorrection3d(self, frames, arbitraryCut=1000): + for module in self.analyzedModules: + frames[module] = self.colCommonModeCorrection(frames[module], arbitraryCut) + return frames + + + def rowCommonModeCorrection(self, frame, arbitraryCut=1000): + ## this takes a 2d object + ## cut keeps photons in common mode - e.g. set to <<1 photon + + for r in range(self.detectorInfo.nRows): + colOffset = 0 + for b in range(0, self.detectorInfo.nBanksCol): + ##for b in range(0, 2): + try: + rowCM = np.median( + frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank][ + frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank] < arbitraryCut + ] + ) + frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank] -= rowCM + except Exception: + rowCM = -666 + print("rowCM problem") + logger.error("rowCM problem") + print(frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank]) + colOffset += self.detectorInfo.nColsPerBank + return frame - self.ds = None - self.det = None ## do we need multiple dets in an array? or self.secondDet? + def colCommonModeCorrection(self, frame, arbitraryCut=1000): + ## this takes a 2d frame + ## cut keeps photons in common mode - e.g. set to <<1 photon - ## for non-120 Hz running - self.nRunCodeEvents = 0 - self.nDaqCodeEvents = 0 - self.nBeamCodeEvents = 0 - self.runCode = 280 - self.daqCode = 281 - self.beamCode = 283 ## per Matt - ##self.beamCode = 281 ## don't see 283... \ No newline at end of file + ##rand = np.random.random() + for c in range(self.detectorInfo.nCols): + rowOffset = 0 + for b in range(0, self.detectorInfo.nBanksRow): + try: + colCM = np.median( + frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c][ + frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c] < arbitraryCut + ] + ) + frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c] -= colCM + except Exception: + colCM = -666 + print("colCM problem") + logger.error("colCM problem") + print(frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank], c) + rowOffset += self.detectorInfo.nRowsPerBank + return frame + + #### End of analysis/correction functions #### \ No newline at end of file diff --git a/calibrationSuite/psana1Base.py b/calibrationSuite/psana1Base.py index 459c75e..46a383f 100755 --- a/calibrationSuite/psana1Base.py +++ b/calibrationSuite/psana1Base.py @@ -7,10 +7,12 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## -# from psana import * -import importlib +""" +This class contains a setup function and some utility functions that work only for psana1 based analysis. +To make the library use this class execute 'export foo=psana1'. +""" + import logging -import os import sys import psana diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index 1155950..8bb004c 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -7,10 +7,12 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## -# from psana import * -import logging +""" +This class contains a setup function and some utility functions that work only for psana2 based analysis. +To make the library use this class execute 'export foo=psana2'. +""" -## for parallelism +import logging import os import sys diff --git a/calibrationSuite/psanaCommon.py b/calibrationSuite/psanaCommon.py index b942f3d..2342e4d 100755 --- a/calibrationSuite/psanaCommon.py +++ b/calibrationSuite/psanaCommon.py @@ -7,13 +7,11 @@ ## may be copied, modified, propagated, or distributed except according to ## the terms contained in the LICENSE.txt file. ############################################################################## - """ -This class has setup-code and variables, along with some utility functions, common for both psana bases. -(psana1Base and psana2Base must both inherit from PsanaCommon!) +This class has setup-code and variables common for both psana bases. +(psana1Base and psana2Base must inherit from this class! ('PsanaCommon')) """ -import psana import logging import importlib.util @@ -36,6 +34,21 @@ def __init__(self, analysisType="scan"): self.args = ArgumentParser().parse_args() logger.info("parsed cmdline args: " + str(self.args)) + self.gainModes = {"FH": 0, "FM": 1, "FL": 2, "AHL-H": 3, "AML-M": 4, "AHL-L": 5, "AML-L": 6} + self.ePix10k_cameraTypes = {1: "Epix10ka", 4: "Epix10kaQuad", 16: "Epix10ka2M"} + + self.ds = None + self.det = None ## do we need multiple dets in an array? or self.secondDet? + + ## for non-120 Hz running + self.nRunCodeEvents = 0 + self.nDaqCodeEvents = 0 + self.nBeamCodeEvents = 0 + self.runCode = 280 + self.daqCode = 281 + self.beamCode = 283 ## per Matt + ##self.beamCode = 281 ## don't see 283... + # Variable-setup calls self.loadExperimentHashFromConfig() self.setupFromExperimentHash() @@ -326,220 +339,4 @@ def setupOutputDirString(self, analysisType): print("output dir: " + self.outputDir) logger.info("output dir: " + self.outputDir) - #### End of setup related functions #### - - #### Start of common getter functions #### - - def get_evrs(self): - if self.config is None: - self.get_config() - - self.evrs = [] - for key in list(self.config.keys()): - if key.type() == psana.EvrData.ConfigV7: - self.evrs.append(key.src()) - - def get_config(self): - self.config = self.ds.env().configStore() - - - def getFivePedestalRunInfo(self): - ## could do load_txt but would require full path so - if self.det is None: - self.setupPsana() - - evt = self.getEvt(self.fivePedestalRun) - self.fpGains = self.det.gain(evt) - self.fpPedestals = self.det.pedestals(evt) - self.fpStatus = self.det.status(evt) ## does this work? - self.fpRMS = self.det.rms(evt) ## does this work? - - def getEvtFromRunsTooSmartForMyOwnGood(self): - for r in self.runRange: - self.run = r - self.ds = self.get_ds() - try: - evt = next(self.ds.events()) - yield evt - except Exception: - continue - - def getEvtFromRuns(self): - try: ## can't get yield to work - evt = next(self.ds.events()) - return evt - except StopIteration: - i = self.runRange.index(self.run) - try: - self.run = self.runRange[i + 1] - print("switching to run %d" % (self.run)) - logger.info("switching to run %d" % (self.run)) - self.ds = self.get_ds(self.run) - except Exception: - print("have run out of new runs") - logger.exception("have run out of new runs") - return None - ##print("get event from new run") - evt = next(self.ds.events()) - return evt - - def getRawData(self, evt, gainBitsMasked=True, negativeGain=False): - frames = self.plainGetRawData(evt) - if frames is None: - return None - if False and self.special: ## turned off for a tiny bit of speed - if "thirteenBits" in self.special: - frames = frames & 0xFFFE - ##print("13bits") - elif "twelveBits" in self.special: - frames = frames & 0xFFFC - ##print("12bits") - elif "elevenBits" in self.special: - frames = frames & 0xFFF8 - ##print("11bits") - elif "tenBits" in self.special: - frames = frames & 0xFFF0 - ##print("10bits") - if self.negativeGain or negativeGain: - zeroPixels = frames == 0 - maskedData = frames & self.gainBitsMask - gainData = frames - maskedData - frames = gainData + self.gainBitsMask - maskedData - frames[zeroPixels] = 0 - if gainBitsMasked: - return frames & self.gainBitsMask - return frames - - def addFakePhotons(self, frames, occupancy, E, width): - shape = frames.shape - occ = np.random.random(shape) - fakes = np.random.normal(E, width, shape) - fakes[occ > occupancy] = 0 - return frames + fakes, (fakes > 0).sum() - - #### End of common getter functions #### - - #### Start of common setter functions #### - - def setROI(self, roiFile=None, roi=None): - """Call with both file name and roi to save roi to file and use, - just name to load, - just roi to set for current job""" - if roiFile is not None: - if roi is None: - self.ROIfile = roiFile - self.ROI = np.load(roiFile) - return - else: - np.save(roiFile, roi) - self.ROI = roi - - #### End of common setter functions #### - - #### Start of common utility functions #### - - def sliceToDetector(self, sliceRow, sliceCol): ## cp from AnalyzeH5: import? - return sliceRow + self.sliceCoordinates[0][0], sliceCol + self.sliceCoordinates[1][0] - - def getNswitchedPixels(self, data, region=None): - return ((data >= self.g0cut) * 1).sum() - - def dumpEventCodeStatistics(self): - print( - "have counted %d run triggers, %d DAQ triggers, %d beam events" - % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) - ) - logger.info( - "have counted %d run triggers, %d DAQ triggers, %d beam events" - % (self.nRunCodeEvents, self.nDaqCodeEvents, self.nBeamCodeEvents) - ) - - def isBeamEvent(self, evt): - if self.ignoreEventCodeCheck: - return True - ec = self.getEventCodes(evt) - ##print(ec[280], ec[281], ec[282], ec[283], ec[284], ec[285] ) - if ec[self.runCode]: - self.nRunCodeEvents += 1 - if ec[self.daqCode]: - self.nDaqCodeEvents += 1 - if self.fakeBeamCode: - return True - if ec[self.beamCode]: - self.nBeamCodeEvents += 1 - return True - ## for FEE, ASC, ... - return self.fakeBeamCode ##False - - #### End of common utility functions #### - - #### Start of analysis/correction functions #### - - def noCommonModeCorrection(self, frames): - return frames - - def regionCommonModeCorrection(self, frame, region, arbitraryCut=1000): - ## this takes a 2d frame - ## cut keeps photons in common mode - e.g. set to <<1 photon - - regionCM = np.median(frame[region][frame[region] < arbitraryCut]) - return frame - regionCM - - def rowCommonModeCorrection3d(self, frames, arbitraryCut=1000): - for module in self.analyzedModules: - frames[module] = self.rowCommonModeCorrection(frames[module], arbitraryCut) - return frames - - def colCommonModeCorrection3d(self, frames, arbitraryCut=1000): - for module in self.analyzedModules: - frames[module] = self.colCommonModeCorrection(frames[module], arbitraryCut) - return frames - - - def rowCommonModeCorrection(self, frame, arbitraryCut=1000): - ## this takes a 2d object - ## cut keeps photons in common mode - e.g. set to <<1 photon - - for r in range(self.detectorInfo.nRows): - colOffset = 0 - for b in range(0, self.detectorInfo.nBanksCol): - ##for b in range(0, 2): - try: - rowCM = np.median( - frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank][ - frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank] < arbitraryCut - ] - ) - frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank] -= rowCM - except Exception: - rowCM = -666 - print("rowCM problem") - logger.error("rowCM problem") - print(frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank]) - colOffset += self.detectorInfo.nColsPerBank - return frame - - def colCommonModeCorrection(self, frame, arbitraryCut=1000): - ## this takes a 2d frame - ## cut keeps photons in common mode - e.g. set to <<1 photon - - ##rand = np.random.random() - for c in range(self.detectorInfo.nCols): - rowOffset = 0 - for b in range(0, self.detectorInfo.nBanksRow): - try: - colCM = np.median( - frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c][ - frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c] < arbitraryCut - ] - ) - frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c] -= colCM - except Exception: - colCM = -666 - print("colCM problem") - logger.error("colCM problem") - print(frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank], c) - rowOffset += self.detectorInfo.nRowsPerBank - return frame - - #### End of analysis/correction functions #### + #### End of setup related functions #### \ No newline at end of file diff --git a/suite_scripts/DeltaTtest.py b/suite_scripts/DeltaTtest.py index cbf2589..9abc8b2 100644 --- a/suite_scripts/DeltaTtest.py +++ b/suite_scripts/DeltaTtest.py @@ -13,7 +13,7 @@ import numpy as np from matplotlib.ticker import AutoMinorLocator -from calibrationSuite.basicSuiteScript import BasicSuiteScript, sortArrayByList +from calibrationSuite.basicSuiteScript import BasicSuiteScript class EventScanParallel(BasicSuiteScript): @@ -133,8 +133,8 @@ def analyze_h5(self, dataFile, label): np.save("%s/pulseIds_c%d_r%d_%s.npy" % (self.outputDir, self.camera, self.run, self.exp), np.array(pulseIds)) dPulseId = pulseIds[1:] - pulseIds[0:-1] - pixels = sortArrayByList(ts, pixels) - rois = sortArrayByList(ts, rois) + pixels = self.sortArrayByList(ts, pixels) + rois = self.sortArrayByList(ts, rois) ts.sort() ts = ts - ts[0] ##ts = ts/np.median(ts[1:]-ts[0:-1]) diff --git a/suite_scripts/EventScanParallelSlice.py b/suite_scripts/EventScanParallelSlice.py index a12de58..30ec207 100644 --- a/suite_scripts/EventScanParallelSlice.py +++ b/suite_scripts/EventScanParallelSlice.py @@ -17,7 +17,7 @@ from matplotlib.ticker import AutoMinorLocator import calibrationSuite.loggingSetup as ls -from calibrationSuite.basicSuiteScript import BasicSuiteScript, sortArrayByList +from calibrationSuite.basicSuiteScript import BasicSuiteScript # for logging from current file logger = logging.getLogger(__name__) @@ -180,9 +180,9 @@ def analyze_h5(self, dataFile, label): dPulseId = pulseIds[1:] - pulseIds[0:-1] # sort pixels and rois based on timestamps - pixels = sortArrayByList(ts, pixels) + pixels = self.sortArrayByList(ts, pixels) if rois is not None: - rois = sortArrayByList(ts, rois) + rois = self.(ts, rois) ts.sort() ts = ts - ts[0] From 80ebbd80dbbc7835f77fa84492577c5e09369746 Mon Sep 17 00:00:00 2001 From: nstelter Date: Wed, 19 Jun 2024 21:02:45 -0700 Subject: [PATCH 14/16] WIP: more refactoring lib code --- suite_scripts/EventScanParallelSlice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suite_scripts/EventScanParallelSlice.py b/suite_scripts/EventScanParallelSlice.py index 30ec207..98387cd 100644 --- a/suite_scripts/EventScanParallelSlice.py +++ b/suite_scripts/EventScanParallelSlice.py @@ -182,7 +182,7 @@ def analyze_h5(self, dataFile, label): # sort pixels and rois based on timestamps pixels = self.sortArrayByList(ts, pixels) if rois is not None: - rois = self.(ts, rois) + rois = self.sortArrayByList(ts, rois) ts.sort() ts = ts - ts[0] From 3404584205e55e420cff989c7e477da2e4103e86 Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Thu, 20 Jun 2024 09:18:37 -0700 Subject: [PATCH 15/16] WIP: more refactoring lib code --- calibrationSuite/basicSuiteScript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index 3cc0d94..513896e 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -146,7 +146,7 @@ def setROI(self, roiFile=None, roi=None): #### Start of common utility functions #### - def sortArrayByList(a, data): + def sortArrayByList(self, a, data): return [x for _, x in sorted(zip(a, data), key=lambda pair: pair[0])] def sliceToDetector(self, sliceRow, sliceCol): ## cp from AnalyzeH5: import? @@ -253,4 +253,4 @@ def colCommonModeCorrection(self, frame, arbitraryCut=1000): rowOffset += self.detectorInfo.nRowsPerBank return frame - #### End of analysis/correction functions #### \ No newline at end of file + #### End of analysis/correction functions #### From 39602676a7f7b0e27754188310abc43a2a65e7b4 Mon Sep 17 00:00:00 2001 From: nstelter Date: Thu, 20 Jun 2024 09:23:21 -0700 Subject: [PATCH 16/16] WIP: more refactoring lib code --- calibrationSuite/basicSuiteScript.py | 15 ++++++----- calibrationSuite/psana1Base.py | 13 +++++----- calibrationSuite/psana2Base.py | 37 +++++++++++++++------------- calibrationSuite/psanaCommon.py | 11 +++++---- 4 files changed, 40 insertions(+), 36 deletions(-) diff --git a/calibrationSuite/basicSuiteScript.py b/calibrationSuite/basicSuiteScript.py index 513896e..1804679 100755 --- a/calibrationSuite/basicSuiteScript.py +++ b/calibrationSuite/basicSuiteScript.py @@ -17,6 +17,7 @@ import os import numpy as np +import psana logger = logging.getLogger(__name__) @@ -27,6 +28,7 @@ print("psana2") from calibrationSuite.psana2Base import PsanaBase + class BasicSuiteScript(PsanaBase): def __init__(self, analysisType="scan"): super().__init__() @@ -35,7 +37,6 @@ def __init__(self, analysisType="scan"): logger.info("in BasicSuiteScript, inheriting from PsanaBase, type is psana%d" % (self.psanaType)) self.className = self.__class__.__name__ - #### Start of common getter functions #### def get_evrs(self): @@ -50,7 +51,6 @@ def get_evrs(self): def get_config(self): self.config = self.ds.env().configStore() - def getFivePedestalRunInfo(self): ## could do load_txt but would require full path so if self.det is None: @@ -61,7 +61,7 @@ def getFivePedestalRunInfo(self): self.fpPedestals = self.det.pedestals(evt) self.fpStatus = self.det.status(evt) ## does this work? self.fpRMS = self.det.rms(evt) ## does this work? - + def getEvtFromRunsTooSmartForMyOwnGood(self): for r in self.runRange: self.run = r @@ -181,21 +181,21 @@ def isBeamEvent(self, evt): return True ## for FEE, ASC, ... return self.fakeBeamCode ##False - + #### End of common utility functions #### #### Start of analysis/correction functions #### def noCommonModeCorrection(self, frames): return frames - + def regionCommonModeCorrection(self, frame, region, arbitraryCut=1000): ## this takes a 2d frame ## cut keeps photons in common mode - e.g. set to <<1 photon regionCM = np.median(frame[region][frame[region] < arbitraryCut]) return frame - regionCM - + def rowCommonModeCorrection3d(self, frames, arbitraryCut=1000): for module in self.analyzedModules: frames[module] = self.rowCommonModeCorrection(frames[module], arbitraryCut) @@ -205,7 +205,6 @@ def colCommonModeCorrection3d(self, frames, arbitraryCut=1000): for module in self.analyzedModules: frames[module] = self.colCommonModeCorrection(frames[module], arbitraryCut) return frames - def rowCommonModeCorrection(self, frame, arbitraryCut=1000): ## this takes a 2d object @@ -252,5 +251,5 @@ def colCommonModeCorrection(self, frame, arbitraryCut=1000): print(frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank], c) rowOffset += self.detectorInfo.nRowsPerBank return frame - + #### End of analysis/correction functions #### diff --git a/calibrationSuite/psana1Base.py b/calibrationSuite/psana1Base.py index 46a383f..aedc211 100755 --- a/calibrationSuite/psana1Base.py +++ b/calibrationSuite/psana1Base.py @@ -16,6 +16,7 @@ import sys import psana + from calibrationSuite.psanaCommon import PsanaCommon logger = logging.getLogger(__name__) @@ -34,11 +35,6 @@ def __init__(self, analysisType="scan"): self.g0cut = 1 << 14 self.gainBitsMask = self.g0cut - 1 - def get_ds(self, run=None): - if run is None: - run = self.run - return psana.DataSource("exp=%s:run=%d:smd" % (self.exp, run)) - def setupPsana(self): logger.info("have built basic script class, exp %s run %d" % (self.exp, self.run)) @@ -60,6 +56,11 @@ def setupPsana(self): except Exception: self.controlData = None + def get_ds(self, run=None): + if run is None: + run = self.run + return psana.DataSource("exp=%s:run=%d:smd" % (self.exp, run)) + def getEvt(self, run=None): oldDs = self.ds if run is not None: @@ -133,4 +134,4 @@ def getCalibData(self, evt): return self.det.calib(evt) def getImage(self, evt, data=None): - return self.raw.image(evt, data) \ No newline at end of file + return self.raw.image(evt, data) diff --git a/calibrationSuite/psana2Base.py b/calibrationSuite/psana2Base.py index 8bb004c..664f963 100755 --- a/calibrationSuite/psana2Base.py +++ b/calibrationSuite/psana2Base.py @@ -16,11 +16,11 @@ import os import sys +import psana ## standard from mpi4py import MPI -import psana from calibrationSuite.psanaCommon import PsanaCommon ##from PSCalib.NDArrIO import load_txt @@ -52,16 +52,13 @@ def __init__(self, analysisType="scan"): self.allowed_timestamp_mismatch = 1000 - def get_ds(self, run=None): - if run is None: - run = self.run - ##tmpDir = '/sdf/data/lcls/ds/rix/rixx1005922/scratch/xtc' - ds = psana.DataSource( - exp=self.exp, run=run, intg_det=self.experimentHash["detectorType"], max_events=self.maxNevents - ) ##, dir=tmpDir) - return ds - def setupPsana(self): + ## fix hardcoding in the fullness of time + self.detEvts = 0 + self.flux = None + self.config = None + self.evrs = None + if self.runRange is None: self.ds = self.get_ds(self.run) else: @@ -93,25 +90,31 @@ def setupPsana(self): self.mfxDg1 = None print("No flux source found") ## if self.verbose? logger.exception("No flux source found") + try: self.mfxDg2 = self.myrun.Detector("MfxDg2BmMon") except Exception: self.mfxDg2 = None - ## fix hardcoding in the fullness of time - self.detEvts = 0 - self.flux = None - self.evrs = None try: self.wave8 = psana.Detector(self.fluxSource, self.ds.env()) except Exception: self.wave8 = None - self.config = None + try: self.controlData = psana.Detector("ControlData") except Exception: self.controlData = None + def get_ds(self, run=None): + if run is None: + run = self.run + ##tmpDir = '/sdf/data/lcls/ds/rix/rixx1005922/scratch/xtc' + ds = psana.DataSource( + exp=self.exp, run=run, intg_det=self.experimentHash["detectorType"], max_events=self.maxNevents + ) ##, dir=tmpDir) + return ds + def getEvtOld(self, run=None): oldDs = self.ds if run is not None: @@ -237,10 +240,10 @@ def plainGetRawData(self, evt): def getCalibData(self, evt): frames = self.det.raw.calib(evt) return frames - + def getImage(self, evt, data=None): return self.det.raw.image(evt, data) - + def getTimestamp(self, evt): return evt.timestamp diff --git a/calibrationSuite/psanaCommon.py b/calibrationSuite/psanaCommon.py index 2342e4d..0c62932 100755 --- a/calibrationSuite/psanaCommon.py +++ b/calibrationSuite/psanaCommon.py @@ -12,12 +12,13 @@ (psana1Base and psana2Base must inherit from this class! ('PsanaCommon')) """ -import logging import importlib.util +import logging ## for parallelism import os import sys + import numpy as np from calibrationSuite.argumentParser import ArgumentParser @@ -170,7 +171,7 @@ def setupFromCmdlineArgs(self): Sets up configurations and parameters based on user-entered command-line arguments. Initializes necessary attributes and handles exceptions on specific evaluations. """ - + self.special = self.args.special if not self.fakeBeamCode: ## defined in ignoreEventCodeCheck @@ -242,7 +243,7 @@ def setupFromCmdlineArgs(self): logger.exception("Error evaluating runRange: " + str(e)) self.runRange = None - self.loadPedestalGainOffsetFiles() + self.loadPedestalGainOffsetFiles() if self.args.detType == "": ## assume epix10k for now @@ -314,7 +315,7 @@ def setupOutputDirString(self, analysisType): The output directory can be specified via a command-line argument, and its location must be specified relative to the 'OUTPUT_ROOT environment variable. If the directory does not exist, it logs error and exits. """ - + # output dir is where we dump .npy, .h5, and .png files self.outputDir = "/%s/" % (analysisType) if self.args.path is not None: @@ -339,4 +340,4 @@ def setupOutputDirString(self, analysisType): print("output dir: " + self.outputDir) logger.info("output dir: " + self.outputDir) - #### End of setup related functions #### \ No newline at end of file + #### End of setup related functions ####