diff --git a/src/pyFAI/app/integrate.py b/src/pyFAI/app/integrate.py index 0426c9ef6..0e8cee99a 100755 --- a/src/pyFAI/app/integrate.py +++ b/src/pyFAI/app/integrate.py @@ -33,7 +33,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "14/02/2025" +__date__ = "17/02/2025" __satus__ = "production" import sys diff --git a/src/pyFAI/diffmap.py b/src/pyFAI/diffmap.py index 770b3da1b..e8f9d6cf4 100644 --- a/src/pyFAI/diffmap.py +++ b/src/pyFAI/diffmap.py @@ -31,7 +31,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "27/01/2025" +__date__ = "19/02/2025" __status__ = "development" __docformat__ = 'restructuredtext' @@ -53,6 +53,7 @@ from . import version as PyFAI_VERSION, date as PyFAI_DATE from .integrator.load_engines import PREFERED_METHODS_2D, PREFERED_METHODS_1D from .io import Nexus, get_isotime, h5py +from .io.integration_config import WorkerConfig from .worker import Worker from .utils.decorators import deprecated, deprecated_warning @@ -380,10 +381,13 @@ def parse(self, sysargv=None, with_config=False): def configure_worker(self, dico=None): """Configure the worker from the dictionary - :param dico: dictionary with the configuration + :param dico: dictionary/WorkerConfig with the configuration :return: worker """ + if isinstance(dico, dict): + dico = WorkerConfig.from_dict(dico) self.worker.set_config(dico or self.poni) + self.init_shape(dico.shape) def makeHDF5(self, rewrite=False): """ @@ -439,7 +443,6 @@ def makeHDF5(self, rewrite=False): process_grp["offset"] = self.offset config = nxs.new_class(process_grp, "configuration", "NXnote") config["type"] = "text/json" - self.init_shape() worker_config = self.worker.get_config() config["data"] = json.dumps(worker_config, indent=2, separators=(",\r\n", ": ")) @@ -497,20 +500,25 @@ def makeHDF5(self, rewrite=False): self.dataset.attrs["title"] = self.nxdata_grp["map"].attrs["title"] = str(self) self.nxs = nxs - def init_shape(self): + def init_shape(self, shape=None): """Initialize the worker with the proper input shape + :param shape: enforce the shape to initialize to :return: shape of the individual frames """ - # if shape of detector undefined: reading the first image to guess it - if self.ai.detector.shape: - shape = self.ai.detector.shape - else: + former_shape = self.worker.ai.detector.shape + try: with fabio.open(self.inputfiles[0]) as fimg: - shape = fimg.data.shape - self.worker.ai.shape = shape - self.worker._shape = shape - self.worker.output = "raw" + new_shape = fimg.data.shape + except: + new_shape = None + shape = new_shape or shape or former_shape + self.worker.ai.shape = shape + self.worker._shape = shape + print(f"reconfigure worker with shape {shape}") + self.worker.reconfig(shape, sync=True) + self.worker.output = "raw" #after reconfig ! + self.worker.safe = False return shape def init_ai(self): @@ -522,17 +530,12 @@ def init_ai(self): self.configure_worker(self.poni) if not self.nxdata_grp: self.makeHDF5(rewrite=False) - shape = self.init_shape() - data = numpy.empty(shape, dtype=numpy.float32) logger.info(f"Initialization of the Azimuthal Integrator using method {self.method}") # enforce initialization of azimuthal integrator - print(self.ai) - res = self.worker.process(data) - tth = res.radial - if self.dataset is None: - self.makeHDF5() + logger.info(f"Detector shape: {self.ai.detector.shape} mask shape {self.ai.detector.mask.shape}") + tth = self.worker.radial - if res.sigma is not None: + if self.worker.propagate_uncertainties: self.dataset_error = self.nxdata_grp.create_dataset("errors", shape=self.dataset.shape, dtype="float32", @@ -542,16 +545,16 @@ def init_ai(self): space = self.unit.space unit = str(self.unit)[len(space) + 1:] if space not in self.nxdata_grp: - self.nxdata_grp[space] = tth - self.nxdata_grp[space].attrs["unit"] = unit - self.nxdata_grp[space].attrs["long_name"] = self.unit.label - self.nxdata_grp[space].attrs["interpretation"] = "scalar" + tthds = self.nxdata_grp.create_dataset(space, data=tth) + tthds.attrs["unit"] = unit + tthds.attrs["long_name"] = self.unit.label + tthds.attrs["interpretation"] = "scalar" if self.worker.do_2D(): - self.nxdata_grp["azimuthal"] = res.azimuthal - self.nxdata_grp["azimuthal"].attrs["unit"] = "deg" - self.nxdata_grp["azimuthal"].attrs["interpretation"] = "scalar" - self.nxdata_grp["azimuthal"].attrs["axes"] = 1 - azim = res.azimuthal + azimds = self.nxdata_grp.create_dataset("azimuthal", data=self.worker.azimuthal) + azimds.attrs["unit"] = "deg" + azimds.attrs["interpretation"] = "scalar" + azimds.attrs["axes"] = 1 + azim = self.worker.azimuthal self.nxdata_grp[space].attrs["axes"] = 2 else: self.nxdata_grp[space].attrs["axes"] = 1 @@ -672,10 +675,9 @@ def process(self): self.makeHDF5() self.init_ai() t0 = -time.perf_counter() - print(self.inputfiles) for f in self.inputfiles: self.process_one_file(f) - t0 -= time.perf_counter() + t0 += time.perf_counter() cnt = max(self._idx, 0) + 1 print(f"Execution time for {cnt} frames: {t0:.3f} s; " f"Average execution time: {1000. * t0 / cnt:.1f} ms/img") diff --git a/src/pyFAI/geometry/core.py b/src/pyFAI/geometry/core.py index 384921c5a..b441c8ebb 100644 --- a/src/pyFAI/geometry/core.py +++ b/src/pyFAI/geometry/core.py @@ -40,7 +40,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "31/01/2025" +__date__ = "19/02/2025" __status__ = "production" __docformat__ = 'restructuredtext' @@ -1357,24 +1357,32 @@ def load(self, filename): """ Load the refined parameters from a file. - :param filename: name of the file to load. Can also be a JSON string with a dict or dict - :type filename: string + :param filename: name of the file to load. Can also be a JSON string with a dict or dict or geometry + :type filename: string with filename or JSON-serialized dict (i.e. string) or a dictionary or Geometry instance. :return: itself with updated parameters """ - try: - if os.path.exists(filename): - with open(filename) as f: - dico = json.load(f) - else: - dico = json.loads(filename) - except Exception: - logger.info("Unable to parse %s as JSON file, defaulting to PoniParser", filename) + poni = None + if isinstance(filename, ponifile.PoniFile): + poni = filename + elif isinstance(filename, (dict, Geometry)): poni = ponifile.PoniFile(data=filename) + elif isinstance(filename, str): + try: + if os.path.exists(filename): + with open(filename) as f: + dico = json.load(f) + else: + dico = json.loads(filename) + except Exception: + logger.info("Unable to parse %s as JSON file, defaulting to PoniParser", filename) + poni = ponifile.PoniFile(data=filename) + else: + config = integration_config.ConfigurationReader(dico) + poni = config.pop_ponifile() else: - config = integration_config.ConfigurationReader(dico) - poni = config.pop_ponifile() - self._init_from_poni(poni) - + logger.error("Unable to initialize geometry from %s", filename) + if poni: + self._init_from_poni(poni) return self read = load @@ -1891,6 +1899,9 @@ def polarization(self, shape=None, factor=None, axis_offset=0, with_checksum=Fal if isinstance(factor, PolarizationDescription): desc = factor factor, axis_offset = desc + elif isinstance(factor, list) and len(factor)==2: + desc = PolarizationDescription(*factor) + factor, axis_offset = desc else: factor = float(factor) axis_offset = float(axis_offset) diff --git a/src/pyFAI/gui/diffmap_widget.py b/src/pyFAI/gui/diffmap_widget.py index 0c312047f..53082751b 100644 --- a/src/pyFAI/gui/diffmap_widget.py +++ b/src/pyFAI/gui/diffmap_widget.py @@ -31,7 +31,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "27/01/2025" +__date__ = "03/02/2025" __status__ = "development" __docformat__ = 'restructuredtext' @@ -155,7 +155,7 @@ class DiffMapWidget(qt.QWidget): def __init__(self): qt.QWidget.__init__(self) - self.integration_config = {} + self.integration_config = None self.list_dataset = ListDataSet() # Contains all datasets to be treated. try: @@ -526,16 +526,15 @@ def process(self, config=None): last_processed_file = None with self.processing_sem: config = self.dump() - config_ai = config.get("ai", {}) - config_ai = config_ai.copy() + config_ai = self.integration_config diffmap_kwargs = {} - diffmap_kwargs["nbpt_rad"] = config_ai.get("nbpt_rad") + diffmap_kwargs["nbpt_rad"] = config_ai.nbpt_rad for key in ["nbpt_fast", "nbpt_slow"]: if key in config: diffmap_kwargs[key] = config[key] - if config_ai.get("do_2D"): - diffmap_kwargs["nbpt_azim"] = config_ai.get("nbpt_azim", 1) + if config_ai.do_2D: + diffmap_kwargs["nbpt_azim"] = config_ai.nbpt_azim diffmap = DiffMap(**diffmap_kwargs) diffmap.inputfiles = [i.path for i in self.list_dataset] # in case generic detector without shape diff --git a/src/pyFAI/gui/pilx/MainWindow.py b/src/pyFAI/gui/pilx/MainWindow.py index b10d54834..55499c22a 100644 --- a/src/pyFAI/gui/pilx/MainWindow.py +++ b/src/pyFAI/gui/pilx/MainWindow.py @@ -33,7 +33,7 @@ __contact__ = "loic.huder@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "27/01/2025" +__date__ = "31/01/2025" __status__ = "development" from typing import Tuple @@ -62,7 +62,7 @@ from .widgets.MapPlotWidget import MapPlotWidget from .widgets.TitleWidget import TitleWidget from ...io.integration_config import WorkerConfig - +logger = logging.getLogger(__name__) class MainWindow(qt.QMainWindow): sigFileChanged = qt.Signal(str) @@ -175,26 +175,24 @@ def initData(self, try: image_grp = h5file[path] except KeyError: - error_msg = f"Cannot access diffraction images at {path}: no such path." - logging.warning(error_msg) - status_bar = self.statusBar() - if status_bar: - status_bar.showMessage(error_msg) + self.warning(f"Cannot access diffraction images at {path}: no such path.") else: - if not isinstance(image_grp, h5py.Group): - error_msg = f"Cannot access diffraction images at {path}: not a group." - logging.warning(error_msg) - status_bar = self.statusBar() - if status_bar: - status_bar.showMessage(error_msg) - else: + if isinstance(image_grp, h5py.Group): lst = [] for key in image_grp: - if key.startswith(base) and isinstance(image_grp[key], h5py.Dataset): - lst.append(key) + try: + ds = image_grp[key] + except KeyError: + self.warning(f"Cannot access diffraction images at {path}/{key}: not a valid dataset.") + else: + if key.startswith(base) and isinstance(ds, h5py.Dataset): + lst.append(key) + lst.sort() for key in lst: self._dataset_paths[posixpath.join(path, key)] = len(image_grp[key]) + else: + self.warning(f"Cannot access diffraction images at {path}: not a group.") self._radial_matrix = compute_radial_values(self.worker_config) self._delta_radial_over_2 = delta_radial / 2 @@ -252,20 +250,20 @@ def displayImageAtIndices(self, indices: ImageIndices): nxprocess = h5file[self._nxprocess_path] map_shape = get_dataset(nxprocess, "result/intensity").shape image_index = row * map_shape[1] + col + self._offset - for dataset_path, size in self._dataset_paths.items(): - if image_index < size: - break - else: - image_index -= size + if self._dataset_paths: + for dataset_path, size in self._dataset_paths.items(): + if image_index < size: + break + else: + image_index -= size + else: + self.warning(f"No diffraction data images found in {self._file_name}") + return try: image_dset = get_dataset(h5file, dataset_path) except KeyError: image_link = h5file.get(dataset_path, getlink=True) - error_msg = f"Cannot access diffraction images at {image_link}" - logging.warning(error_msg) - status_bar = self.statusBar() - if status_bar: - status_bar.showMessage(error_msg) + self.warning(f"Cannot access diffraction images at {image_link}") return if image_index >= len(image_dset): @@ -464,3 +462,13 @@ def setNewBackgroundCurve(self, x: float, y: float): def clearPoints(self): for indices in self._fixed_indices.copy(): self.removeMapPoint(indices=indices) + + def warning(self, error_msg): + """Log a warning both in the terminal and in the status bar if possible + + :param error_msg: string with the message + """ + logger.warning(error_msg) + status_bar = self.statusBar() + if status_bar: + status_bar.showMessage(error_msg) diff --git a/src/pyFAI/gui/widgets/WorkerConfigurator.py b/src/pyFAI/gui/widgets/WorkerConfigurator.py index 693126336..81f73ec87 100644 --- a/src/pyFAI/gui/widgets/WorkerConfigurator.py +++ b/src/pyFAI/gui/widgets/WorkerConfigurator.py @@ -33,7 +33,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "07/02/2025" +__date__ = "18/02/2025" __status__ = "development" import logging @@ -255,7 +255,7 @@ def splitFiles(filenames): return [name.strip() for name in filenames.split("|")] wc = integration_config.WorkerConfig(application="pyfai-integrate", - poni=self.getPoniDict()) + poni=self.getPoni()) # pre-processing if self.do_mask.isChecked(): wc.mask_image = str_(self.mask_file.text()).strip() @@ -273,7 +273,7 @@ def splitFiles(filenames): if self.do_2D.isChecked(): wc.nbpt_azim = self.__getAzimuthalNbpt() wc.nbpt_rad = self.__getRadialNbpt() - wc.unit = str(self.radial_unit.model().value()) + wc.unit = to_unit(str(self.radial_unit.model().value())) if self.do_radial_range.isChecked(): wc.radial_range = [self._float("radial_range_min", -numpy.inf), self._float("radial_range_max", numpy.inf)] @@ -361,23 +361,23 @@ def normalizeFiles(filenames): self.__geometryModel.rotation3().setValue(None) poni = wc.poni - if isinstance(poni, PoniFile): - if poni.wavelength: - self.__geometryModel.wavelength().setValue(poni.wavelength) - if poni.dist: - self.__geometryModel.distance().setValue(poni.dist) - if poni.poni1 is not None: - self.__geometryModel.poni1().setValue(poni.poni1) - if poni.poni2 is not None: - self.__geometryModel.poni2().setValue(poni.poni2) - if poni.rot1 is not None: - self.__geometryModel.rotation1().setValue(poni.rot1) - if poni.rot2 is not None: - self.__geometryModel.rotation2().setValue(poni.rot2) - if poni.rot3 is not None: - self.__geometryModel.rotation3().setValue(poni.rot3) - - # reader = integration_config.ConfigurationReader(dico) + if not isinstance(poni, PoniFile): + poni = PoniFile(poni) + + if poni.wavelength: + self.__geometryModel.wavelength().setValue(poni.wavelength) + if poni.dist: + self.__geometryModel.distance().setValue(poni.dist) + if poni.poni1 is not None: + self.__geometryModel.poni1().setValue(poni.poni1) + if poni.poni2 is not None: + self.__geometryModel.poni2().setValue(poni.poni2) + if poni.rot1 is not None: + self.__geometryModel.rotation1().setValue(poni.rot1) + if poni.rot2 is not None: + self.__geometryModel.rotation2().setValue(poni.rot2) + if poni.rot3 is not None: + self.__geometryModel.rotation3().setValue(poni.rot3) # detector if poni.detector is not None: diff --git a/src/pyFAI/integrator/azimuthal.py b/src/pyFAI/integrator/azimuthal.py index cdbd36b4c..01371b9ec 100644 --- a/src/pyFAI/integrator/azimuthal.py +++ b/src/pyFAI/integrator/azimuthal.py @@ -4,7 +4,7 @@ # Project: Azimuthal integration # https://github.com/silx-kit/pyFAI # -# Copyright (C) 2012-2024 European Synchrotron Radiation Facility, Grenoble, France +# Copyright (C) 2012-2025 European Synchrotron Radiation Facility, Grenoble, France # # Principal author: Jérôme Kieffer (Jerome.Kieffer@ESRF.eu) # @@ -30,7 +30,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "31/01/2025" +__date__ = "19/02/2025" __status__ = "stable" __docformat__ = 'restructuredtext' @@ -43,7 +43,7 @@ # from ..geometry import Geometry from .. import units from ..utils import EPS32, deg2rad, crc32, rad2rad -# from ..utils.decorators import deprecated, deprecated_warning +from ..utils.mathutil import nan_equal from ..containers import Integrate1dResult, Integrate2dResult, SeparateResult, ErrorModel from ..io import DefaultAiWriter, save_integrate_result from ..io.ponifile import PoniFile @@ -128,7 +128,7 @@ def integrate1d(self, data, npt, filename=None, method = self._normalize_method(method, dim=1, default=self.DEFAULT_METHOD_1D) assert method.dimension == 1 unit = units.to_unit(unit) - empty = dummy if dummy is not None else self._empty + empty = numpy.float32(dummy) if dummy is not None else self._empty shape = data.shape pos0_scale = unit.scale @@ -206,23 +206,23 @@ def integrate1d(self, data, npt, filename=None, if (not cython_reset) and safe: if cython_integr.unit != unit: cython_reset = "unit was changed" - if cython_integr.bins != npt: + elif cython_integr.bins != npt: cython_reset = "number of points changed" - if cython_integr.size != data.size: + elif cython_integr.size != data.size: cython_reset = "input image size changed" - if cython_integr.empty != empty: - cython_reset = "empty value changed" - if (mask is not None) and (not cython_integr.check_mask): + elif not nan_equal(cython_integr.empty, empty): + cython_reset = f"empty value changed {cython_integr.empty}!={empty}" + elif (mask is not None) and (not cython_integr.check_mask): cython_reset = f"mask but {method.algo_lower.upper()} was without mask" elif (mask is None) and (cython_integr.cmask is not None): cython_reset = f"no mask but { method.algo_lower.upper()} has mask" elif (mask is not None) and (cython_integr.mask_checksum != mask_crc): cython_reset = "mask changed" - if (radial_range is None) and (cython_integr.pos0_range is not None): + elif (radial_range is None) and (cython_integr.pos0_range is not None): cython_reset = f"radial_range was defined in { method.algo_lower.upper()}" elif (radial_range is not None) and (cython_integr.pos0_range != radial_range): cython_reset = f"radial_range is defined but differs in %s" % method.algo_lower.upper() - if (azimuth_range is None) and (cython_integr.pos1_range is not None): + elif (azimuth_range is None) and (cython_integr.pos1_range is not None): cython_reset = f"azimuth_range not defined and {method.algo_lower.upper()} had azimuth_range defined" elif (azimuth_range is not None) and (cython_integr.pos1_range != azimuth_range): cython_reset = f"azimuth_range requested and {method.algo_lower.upper()}'s azimuth_range don't match" @@ -275,23 +275,23 @@ def integrate1d(self, data, npt, filename=None, if (not reset) and safe: if integr.unit != unit: reset = "unit was changed" - if integr.bins != npt: + elif integr.bins != npt: reset = "number of points changed" - if integr.size != data.size: + elif integr.size != data.size: reset = "input image size changed" - if integr.empty != empty: - reset = "empty value changed" - if (mask is not None) and (not integr.check_mask): + elif not nan_equal(integr.empty, empty): + reset = f"empty value changed {integr.empty}!={empty}" + elif (mask is not None) and (not integr.check_mask): reset = f"mask but {method.algo_lower.upper()} was without mask" elif (mask is None) and (integr.check_mask): reset = f"no mask but {method.algo_lower.upper()} has mask" elif (mask is not None) and (integr.mask_checksum != mask_crc): reset = "mask changed" - if (radial_range is None) and (integr.pos0_range is not None): + elif (radial_range is None) and (integr.pos0_range is not None): reset = f"radial_range was defined in {method.algo_lower.upper()}" elif (radial_range is not None) and (integr.pos0_range != radial_range): reset = f"radial_range is defined but differs in {method.algo_lower.upper()}" - if (azimuth_range is None) and (integr.pos1_range is not None): + elif (azimuth_range is None) and (integr.pos1_range is not None): reset = f"azimuth_range not defined and {method.algo_lower.upper()} had azimuth_range defined" elif (azimuth_range is not None) and (integr.pos1_range != azimuth_range): reset = f"azimuth_range requested and {method.algo_lower.upper()}'s azimuth_range don't match" @@ -423,12 +423,12 @@ def integrate1d(self, data, npt, filename=None, if (not reset) and safe: if integr.unit != unit: reset = "unit was changed" - if integr.bins != npt: + elif integr.bins != npt: reset = "number of points changed" - if integr.size != data.size: + elif integr.size != data.size: reset = "input image size changed" - if integr.empty != empty: - reset = "empty value changed" + elif not nan_equal(integr.empty, empty): + reset = f"empty value changed {integr.empty}!={empty}" if reset: logger.info("ai.integrate1d: Resetting integrator because %s", reset) pos0 = self.array_from_unit(shape, "center", unit, scale=False) @@ -605,7 +605,7 @@ def integrate_radial(self, data, npt, npt_rad=100, sum_normalization = res._sum_normalization.sum(axis=-1) mask = numpy.where(count == 0) - empty = dummy if dummy is not None else self._empty + empty = numpy.float32(dummy) if dummy is not None else self._empty intensity = sum_signal / sum_normalization intensity[mask] = empty @@ -703,7 +703,7 @@ def integrate2d_ng(self, data, npt_rad, npt_azim=360, space = (radial_unit.space, azimuth_unit.space) pos0_scale = radial_unit.scale pos1_scale = azimuth_unit.scale - empty = dummy if dummy is not None else self._empty + empty = numpy.float32(dummy) if dummy is not None else self._empty if mask is None: has_mask = "from detector" mask = self.mask @@ -799,8 +799,8 @@ def integrate2d_ng(self, data, npt_rad, npt_azim=360, cython_reset = f"number of points {cython_integr.bins} incompatible with requested {npt}" if cython_integr.size != data.size: cython_reset = f"input image size {cython_integr.size} incompatible with requested {data.size}" - if cython_integr.empty != empty: - cython_reset = f"empty value {cython_integr.empty} incompatible with requested {empty}" + if not nan_equal(cython_integr.empty, empty): + cython_reset = f"empty value changed {cython_integr.empty}!={empty}" if (mask is not None) and (not cython_integr.check_mask): cython_reset = f"mask but {method.algo_lower.upper()} was without mask" elif (mask is None) and (cython_integr.cmask is not None): @@ -1450,23 +1450,23 @@ def medfilt1d_ng(self, data, if (not cython_reset) and safe: if cython_integr.unit != unit: cython_reset = "unit was changed" - if cython_integr.bins != npt: + elif cython_integr.bins != npt: cython_reset = "number of points changed" - if cython_integr.size != data.size: + elif cython_integr.size != data.size: cython_reset = "input image size changed" - if cython_integr.empty != self._empty: - cython_reset = "empty value changed " - if (mask is not None) and (not cython_integr.check_mask): + elif not nan_equal(cython_integr.empty, self._empty): + cython_reset = f"empty value changed {cython_integr.empty}!={self._empty}" + elif (mask is not None) and (not cython_integr.check_mask): cython_reset = "mask but CSR was without mask" elif (mask is None) and (cython_integr.check_mask): cython_reset = "no mask but CSR has mask" elif (mask is not None) and (cython_integr.mask_checksum != mask_crc): cython_reset = "mask changed" - if (radial_range is None) and (cython_integr.pos0_range is not None): + elif (radial_range is None) and (cython_integr.pos0_range is not None): cython_reset = "radial_range was defined in CSR" elif (radial_range is not None) and cython_integr.pos0_range != (min(radial_range), max(radial_range)): cython_reset = "radial_range is defined but not the same as in CSR" - if (azimuth_range is None) and (cython_integr.pos1_range is not None): + elif (azimuth_range is None) and (cython_integr.pos1_range is not None): cython_reset = "azimuth_range not defined and CSR had azimuth_range defined" elif (azimuth_range is not None) and cython_integr.pos1_range != (min(azimuth_range), max(azimuth_range)): cython_reset = "azimuth_range requested and CSR's azimuth_range don't match" @@ -1507,24 +1507,23 @@ def medfilt1d_ng(self, data, if (not reset) and safe: if integr.unit != unit: reset = "unit was changed" - if integr.bins != npt: + elif integr.bins != npt: reset = "number of points changed" - if integr.size != data.size: + elif integr.size != data.size: reset = "input image size changed" - if integr.empty != self._empty: - reset = "empty value changed " - if (mask is not None) and (not integr.check_mask): + elif not nan_equal(integr.empty, self._empty): + reset = f"empty value changed {integr.empty}!={self._empty}" + elif (mask is not None) and (not integr.check_mask): reset = "mask but CSR was without mask" elif (mask is None) and (integr.check_mask): reset = "no mask but CSR has mask" elif (mask is not None) and (integr.mask_checksum != mask_crc): reset = "mask changed" - # TODO - if (radial_range is None) and (integr.pos0_range is not None): + elif (radial_range is None) and (integr.pos0_range is not None): reset = "radial_range was defined in CSR" elif (radial_range is not None) and integr.pos0_range != (min(radial_range), max(radial_range)): reset = "radial_range is defined but not the same as in CSR" - if (azimuth_range is None) and (integr.pos1_range is not None): + elif (azimuth_range is None) and (integr.pos1_range is not None): reset = "azimuth_range not defined and CSR had azimuth_range defined" elif (azimuth_range is not None) and integr.pos1_range != (min(azimuth_range), max(azimuth_range)): reset = "azimuth_range requested and CSR's azimuth_range don't match" @@ -1910,23 +1909,23 @@ def sigma_clip(self, data, if (not cython_reset) and safe: if cython_integr.unit != unit: cython_reset = "unit was changed" - if cython_integr.bins != npt: + elif cython_integr.bins != npt: cython_reset = "number of points changed" - if cython_integr.size != data.size: + elif cython_integr.size != data.size: cython_reset = "input image size changed" - if cython_integr.empty != self._empty: - cython_reset = "empty value changed " - if (mask is not None) and (not cython_integr.check_mask): + elif not nan_equal(cython_integr.empty, self._empty): + cython_reset = f"empty value changed {cython_integr.empty}!={self._empty}" + elif (mask is not None) and (not cython_integr.check_mask): cython_reset = "mask but CSR was without mask" elif (mask is None) and (cython_integr.check_mask): cython_reset = "no mask but CSR has mask" elif (mask is not None) and (cython_integr.mask_checksum != mask_crc): cython_reset = "mask changed" - if (radial_range is None) and (cython_integr.pos0_range is not None): + elif (radial_range is None) and (cython_integr.pos0_range is not None): cython_reset = "radial_range was defined in CSR" elif (radial_range is not None) and cython_integr.pos0_range != (min(radial_range), max(radial_range)): cython_reset = "radial_range is defined but not the same as in CSR" - if (azimuth_range is None) and (cython_integr.pos1_range is not None): + elif (azimuth_range is None) and (cython_integr.pos1_range is not None): cython_reset = "azimuth_range not defined and CSR had azimuth_range defined" elif (azimuth_range is not None) and cython_integr.pos1_range != (min(azimuth_range), max(azimuth_range)): cython_reset = "azimuth_range requested and CSR's azimuth_range don't match" @@ -1967,24 +1966,23 @@ def sigma_clip(self, data, if (not reset) and safe: if integr.unit != unit: reset = "unit was changed" - if integr.bins != npt: + elif integr.bins != npt: reset = "number of points changed" - if integr.size != data.size: + elif integr.size != data.size: reset = "input image size changed" - if integr.empty != self._empty: - reset = "empty value changed " - if (mask is not None) and (not integr.check_mask): + elif not nan_equal(integr.empty, self._empty): + reset = f"empty value changed {integr.empty}!={self._empty}" + elif (mask is not None) and (not integr.check_mask): reset = "mask but CSR was without mask" elif (mask is None) and (integr.check_mask): reset = "no mask but CSR has mask" elif (mask is not None) and (integr.mask_checksum != mask_crc): reset = "mask changed" - # TODO - if (radial_range is None) and (integr.pos0_range is not None): + elif (radial_range is None) and (integr.pos0_range is not None): reset = "radial_range was defined in CSR" elif (radial_range is not None) and integr.pos0_range != (min(radial_range), max(radial_range)): reset = "radial_range is defined but not the same as in CSR" - if (azimuth_range is None) and (integr.pos1_range is not None): + elif (azimuth_range is None) and (integr.pos1_range is not None): reset = "azimuth_range not defined and CSR had azimuth_range defined" elif (azimuth_range is not None) and integr.pos1_range != (min(azimuth_range), max(azimuth_range)): reset = "azimuth_range requested and CSR's azimuth_range don't match" diff --git a/src/pyFAI/integrator/common.py b/src/pyFAI/integrator/common.py index 7262cfda2..cb8866c35 100644 --- a/src/pyFAI/integrator/common.py +++ b/src/pyFAI/integrator/common.py @@ -4,7 +4,7 @@ # Project: Azimuthal integration # https://github.com/silx-kit/pyFAI # -# Copyright (C) 2012-2024 European Synchrotron Radiation Facility, Grenoble, France +# Copyright (C) 2012-2025 European Synchrotron Radiation Facility, Grenoble, France # # Principal author: Jérôme Kieffer (Jerome.Kieffer@ESRF.eu) # @@ -30,7 +30,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "10/10/2024" +__date__ = "18/02/2025" __status__ = "stable" __docformat__ = 'restructuredtext' @@ -138,7 +138,7 @@ def __init__(self, dist=1, poni1=0, poni2=0, self._lock = threading.Semaphore() self.engines = {} # key: name of the engine, - self._empty = 0.0 + self._empty = numpy.float32(0.0) def reset(self, collect_garbage=True): """Reset azimuthal integrator in addition to other arrays. @@ -1707,15 +1707,16 @@ def get_empty(self): return self._empty def set_empty(self, value): - self._empty = float(value) + value = value or 0.0 + self._empty = numpy.float32(value) # propagate empty values to integrators for engine in self.engines.values(): with engine.lock: if engine.engine is not None: try: engine.engine.empty = self._empty - except Exception as exeption: - logger.error(exeption) + except Exception as exception: + logger.error(f"{type(exception)}: {exception}") empty = property(get_empty, set_empty) diff --git a/src/pyFAI/io/__init__.py b/src/pyFAI/io/__init__.py index 6fcef9203..4d02d1348 100644 --- a/src/pyFAI/io/__init__.py +++ b/src/pyFAI/io/__init__.py @@ -42,7 +42,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "07/02/2025" +__date__ = "17/02/2025" __status__ = "production" __docformat__ = 'restructuredtext' @@ -108,7 +108,7 @@ def __init__(self, filename=None, extension=None): self.lima_cfg = {} def __repr__(self): - return "Generic writer on file %s" % (self.filename) + return f"Generic writer on file {self.filename}" def init(self, fai_cfg=None, lima_cfg=None): """ @@ -217,7 +217,7 @@ def __init__(self, filename, hpath=None, entry_template=None, fast_scan_width=No self._mode = mode def __repr__(self): - return "HDF5 writer on file {self.filename}:{self.hpath} {'' if self._initialized else 'un'}initialized {}" + return f"HDF5 writer on file {self.filename}:{self.hpath} {'' if self.intensity_ds else 'un'}initialized" def _require_main_entry(self, mode): """ @@ -328,9 +328,7 @@ def init(self, fai_cfg=None, lima_cfg=None): self.radial_ds.attrs["name"] = rad_name self.radial_ds.attrs["long_name"] = "Diffraction radial direction %s (%s)" % (rad_name, rad_unit) - self.do2D = self.fai_cfg.do_2D - - if self.do2D: + if self.fai_cfg.do_2D: self.azimuthal_ds = self.nxdata_grp.require_dataset("chi", (self.fai_cfg.nbpt_azim,), numpy.float32) self.azimuthal_ds.attrs["unit"] = "deg" self.azimuthal_ds.attrs["interpretation"] = "scalar" @@ -343,7 +341,7 @@ def init(self, fai_cfg=None, lima_cfg=None): self.fast_motor = self.entry_grp.require_dataset("fast", (self.fast_scan_width,), numpy.float32) self.fast_motor.attrs["long_name"] = "Fast motor position" self.fast_motor.attrs["interpretation"] = "scalar" - if self.do2D: + if self.fai_cfg.do_2D: chunk = 1, self.fast_scan_width, self.fai_cfg.nbpt_azim, self.fai_cfg.nbpt_rad self.ndim = 4 axis_definition = [".", "fast", "chi", "radial"] @@ -352,7 +350,7 @@ def init(self, fai_cfg=None, lima_cfg=None): self.ndim = 3 axis_definition = [".", "fast", "radial"] else: - if self.do2D: + if self.fai_cfg.do_2D: axis_definition = [".", "chi", "radial"] chunk = 1, self.fai_cfg["nbpt_azim"], self.fai_cfg.nbpt_rad self.ndim = 3 diff --git a/src/pyFAI/io/integration_config.py b/src/pyFAI/io/integration_config.py index de4a6bb21..01ba0e831 100644 --- a/src/pyFAI/io/integration_config.py +++ b/src/pyFAI/io/integration_config.py @@ -757,12 +757,6 @@ def __setitem__(self, key, value): @decorators.deprecated(reason="WorkerConfig now dataclass, no more a dict", replacement=None, since_version="2025.01") def __getitem__(self, key): return self.__getattribute__(key) - @decorators.deprecated(reason="WorkerConfig now dataclass, no more a dict", replacement=None, since_version="2025.01") - def get(self, key, default=None): - try: - return self.__getattribute__(key) - except: - return default @decorators.deprecated(reason="WorkerConfig now dataclass, no more a dict", replacement=None, since_version="2025.01") def __contains__(self, key): diff --git a/src/pyFAI/io/ponifile.py b/src/pyFAI/io/ponifile.py index 4f9316c7f..38fbfc848 100644 --- a/src/pyFAI/io/ponifile.py +++ b/src/pyFAI/io/ponifile.py @@ -31,7 +31,7 @@ __author__ = "Jérôme Kieffer" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "07/02/2025" +__date__ = "17/02/2025" __docformat__ = 'restructuredtext' import collections @@ -173,18 +173,13 @@ def read_from_dict(self, config): self._dist = float(config["distance"]) if config["distance"] is not None else None elif "dist" in config: self._dist = float(config["dist"]) if config["dist"] is not None else None - if "poni1" in config: - self._poni1 = float(config["poni1"]) if config["poni1"] is not None else None - if "poni2" in config: - self._poni2 = float(config["poni2"]) if config["poni2"] is not None else None - if "rot1" in config: - self._rot1 = float(config["rot1"]) if config["rot1"] is not None else None - if "rot2" in config: - self._rot2 = float(config["rot2"]) if config["rot2"] is not None else None - if "rot3" in config: - self._rot3 = float(config["rot3"]) if config["rot3"] is not None else None - if "wavelength" in config: - self._wavelength = float(config["wavelength"]) if config["wavelength"] is not None else None + + self._poni1 = float(config["poni1"]) if config.get("poni1") is not None else None + self._poni2 = float(config["poni2"]) if config.get("poni2") is not None else None + self._rot1 = float(config["rot1"]) if config.get("rot1") is not None else None + self._rot2 = float(config["rot2"]) if config.get("rot2") is not None else None + self._rot3 = float(config["rot3"]) if config.get("rot3") is not None else None + self._wavelength = float(config["wavelength"]) if config.get("wavelength") is not None else None def read_from_duck(self, duck): """Initialize the object using an object providing the same API. diff --git a/src/pyFAI/test/test_integrate_app.py b/src/pyFAI/test/test_integrate_app.py index dfbdc71b3..995c530b8 100644 --- a/src/pyFAI/test/test_integrate_app.py +++ b/src/pyFAI/test/test_integrate_app.py @@ -90,7 +90,7 @@ def create_json(self, ponipath=None, nbpt_azim=1): ponipath = UtilsTest.getimage("dummy.poni") data = {"poni": ponipath} integration_config.normalize(data, inplace=True) - data["wavelength"] = 1 + # data["wavelength"] = 1 data["nbpt_rad"] = 3 data["nbpt_azim"] = nbpt_azim data["do_2D"] = nbpt_azim > 1 diff --git a/src/pyFAI/test/test_utils_mathutil.py b/src/pyFAI/test/test_utils_mathutil.py index 2e6465ee3..7096d31d2 100644 --- a/src/pyFAI/test/test_utils_mathutil.py +++ b/src/pyFAI/test/test_utils_mathutil.py @@ -32,7 +32,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "21/01/2025" +__date__ = "19/02/2025" import unittest import numpy @@ -172,6 +172,15 @@ def test_quality_of_fit(self): res = mathutil.quality_of_fit(img, ai, cal, rings=[0,1], npt_azim=36, npt_rad=100) self.assertLess(res, 0.3, "Fit of good quality") + def test_nan_equal(self): + nan_equal = mathutil.nan_equal + self.assertTrue(nan_equal(5, 5.0), "Expected") + self.assertFalse(nan_equal(5, 0), "Nop") + self.assertFalse(nan_equal(5, numpy.nan), "Nop") + self.assertFalse(nan_equal(numpy.nan,5), "Nop") + self.assertTrue(nan_equal(numpy.nan, numpy.nan), "Expected") + + def suite(): loader = unittest.defaultTestLoader.loadTestsFromTestCase testsuite = unittest.TestSuite() diff --git a/src/pyFAI/test/test_worker.py b/src/pyFAI/test/test_worker.py index 262ab3767..b7e097940 100644 --- a/src/pyFAI/test/test_worker.py +++ b/src/pyFAI/test/test_worker.py @@ -4,7 +4,7 @@ # Project: Azimuthal integration # https://github.com/silx-kit/pyFAI # -# Copyright (C) 2015-2024 European Synchrotron Radiation Facility, Grenoble, France +# Copyright (C) 2015-2025 European Synchrotron Radiation Facility, Grenoble, France # # Principal author: Jérôme Kieffer (Jerome.Kieffer@ESRF.eu) # @@ -32,7 +32,7 @@ __contact__ = "valentin.valls@esrf.fr" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "06/02/2025" +__date__ = "17/02/2025" import unittest import logging diff --git a/src/pyFAI/utils/mathutil.py b/src/pyFAI/utils/mathutil.py index 3752044e7..d84f40808 100644 --- a/src/pyFAI/utils/mathutil.py +++ b/src/pyFAI/utils/mathutil.py @@ -34,7 +34,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "21/01/2025" +__date__ = "19/02/2025" __status__ = "production" import logging @@ -1015,3 +1015,10 @@ def quality_of_fit(img, ai, calibrant, idx_fwhm[idx, ring] = width idx_maxi[idx, ring] = idx_max return numpy.nanmean((2.355*(idx_maxi-idx_theo)/idx_fwhm)**2) + + +def nan_equal(a, b): + """return True if a==b, also if a and b are both NaNs""" + if a==b: + return True + return numpy.isnan(a) and numpy.isnan(b) diff --git a/src/pyFAI/worker.py b/src/pyFAI/worker.py index 0746d8e0d..4725decda 100644 --- a/src/pyFAI/worker.py +++ b/src/pyFAI/worker.py @@ -45,7 +45,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "14/02/2025" +__date__ = "19/02/2025" __status__ = "development" import threading @@ -60,7 +60,7 @@ from . import method_registry from .integrator.azimuthal import AzimuthalIntegrator from .containers import ErrorModel -from .method_registry import IntegrationMethod +from .method_registry import IntegrationMethod, Method from .distortion import Distortion from . import units from .io import ponifile, image as io_image @@ -195,7 +195,12 @@ def __init__(self, azimuthalIntegrator=None, self.integrator_name = integrator_name self._processor = None self._nbpt_azim = None - self.method = method + if isinstance(method, (str, list, tuple, Method)): + method = IntegrationMethod.parse(method) + else: + logger.error(f"Unable to parse method {method}") + self.method = (method.split, method.algorithm, method.implementation) + self.opencl_device = method.target self._method = None self.nbpt_azim, self.nbpt_rad = shapeOut self._unit = units.to_unit(unit) @@ -213,6 +218,7 @@ def __init__(self, azimuthalIntegrator=None, self._shape = shapeIn self.radial = None self.azimuthal = None + self.propagate_uncertainties = None self.safe = True self.extra_options = {} if extra_options is None else extra_options.copy() self.radial_range = self.extra_options.pop("radial_range", None) @@ -254,8 +260,29 @@ def update_processor(self, integrator_name=None): elif "1d" in integrator_name and dim == 2: integrator_name = integrator_name.replace("1d", "2d") self._processor = self.ai.__getattribute__(integrator_name) - self._method = IntegrationMethod.select_one_available(self.method, dim=dim) + + if isinstance(self.method, (list, tuple)): + if isinstance(self.method, Method): + methods = IntegrationMethod.select_method(dim=dim, split=self.method[1], algo=self.method[2], impl=self.method[3], + target=self.opencl_device if isinstance(self.opencl_device, (tuple, list)) else self.method[4], + target_type=self.opencl_device if isinstance(self.opencl_device, str) else self.method[4], + degradable=True) + else: + methods = IntegrationMethod.select_method(dim=dim, split=self.method[0], algo=self.method[1], impl=self.method[2], + target=self.opencl_device if isinstance(self.opencl_device, (tuple, list)) else None, + target_type=self.opencl_device if isinstance(self.opencl_device, str) else None, + degradable=True) + self._method = methods[0] + elif isinstance(self.method, str) or self.method is None: + self._method = IntegrationMethod.select_one_available(method=self.method, dim=dim) + elif isinstance(self.method, IntegrationMethod): + self._method = self.method + else: + logger.error(f"No method available for {dim}D integration on {self.method} with target {self.opencl_device}") + self._method = IntegrationMethod.select_one_available(method=self.method, dim=dim) self.integrator_name = self._processor.__name__ + self.radial = None + self.azimuthal = None @property def nbpt_azim(self): @@ -296,6 +323,7 @@ def reconfig(self, shape=None, sync=False): logger.info(f"reconfig: mask has been rebinned from {mask.shape} to {self.ai.detector.mask.shape}. Masking {self.ai.detector.mask.sum()} pixels") else: self.ai.detector.shape = shape + self.ai.empty = self.dummy self.ai.reset() self.warmup(sync) @@ -346,22 +374,10 @@ def process(self, data, variance=None, if self.azimuth_range is not None: kwarg["azimuth_range"] = self.azimuth_range - error = None try: integrated_result = self._processor(**kwarg) - if self.do_2D(): - self.radial = integrated_result.radial - self.azimuthal = integrated_result.azimuthal - result = integrated_result.intensity - if integrated_result.sigma is not None: - error = integrated_result.sigma - else: - self.radial = integrated_result.radial - self.azimuthal = None - result = numpy.vstack(integrated_result).T - except Exception as err: - logger.debug("Backtrace", exc_info=True) + logger.info("Backtrace", exc_info=True) err2 = [f"error in integration do_2d: {self.do_2D()}", str(err.__class__.__name__), str(err), @@ -374,17 +390,23 @@ def process(self, data, variance=None, ] logger.error("\n".join(err2)) raise err - + else: + if self.radial is None: + self.radial = integrated_result.radial + if self.do_2D(): + self.azimuthal = integrated_result.azimuthal if writer is not None: writer.write(integrated_result) - if self.output == "raw": return integrated_result elif self.output == "numpy": - if (variance is not None) and (error is not None): - return result, error + if self.do_2D(): + if integrated_result.sigma is None: + return integrated_result.intensity + else: + return integrated_result.intensity, integrated_result.sigma else: - return result + return numpy.vstack(integrated_result).T def setSubdir(self, path): """ @@ -462,6 +484,7 @@ def set_config(self, config:dict | WorkerConfig, consume_keys:bool=False): self._nbpt_azim = int(config.nbpt_azim) if config.nbpt_azim else 1 self.method = config.method # expand to Method ? + self.opencl_device = config.opencl_device self.nbpt_rad = config.nbpt_rad self.unit = units.to_unit(config.unit or "2th_deg") self.error_model = ErrorModel.parse(config.error_model) @@ -497,10 +520,12 @@ def get_worker_config(self): config = WorkerConfig(application="worker", poni=dict(self.ai.get_config()), unit=str(self._unit)) - for key in ["nbpt_azim", "nbpt_rad", "polarization_factor", "delta_dummy", "extra_options", - "correct_solid_angle", "error_model", "method", "azimuth_range", "radial_range", - "dummy", "normalization_factor", "dark_current_image", "flat_field_image", - "mask_image", "integrator_name"]: + for key in ["nbpt_azim", "nbpt_rad", "polarization_factor", "extra_options", + "correct_solid_angle", "error_model", "method", "opencl_device", + "azimuth_range", "radial_range", + "dummy", "delta_dummy", "normalization_factor", + "dark_current_image", "flat_field_image", + "mask_image", "integrator_name", "shape"]: try: config.__setattr__(key, self.__getattribute__(key)) except Exception as err: @@ -532,6 +557,24 @@ def save_config(self, filename=None): """Save the configuration as a JSON file""" self.get_worker_config().save(filename or self.config_file) + def sync_init(self): + + self.dummy_output = self.process(numpy.zeros(self.shape, dtype=numpy.float32)) + + def _warmup(self): + backup = self.output + self.output = "raw" + msk = "" if self.ai.detector.mask is None else f"mask {self.ai.detector.mask.shape} mask sum {self.ai.detector.mask.sum()}\n" + logger.info(f"warm-up with shape {self.shape}, detector shape {self.ai.detector.shape}\n{msk}{self.ai}") + integrated_result = self.process(numpy.zeros(self.shape, dtype=numpy.float32)) + self.radial = integrated_result.radial + if self.do_2D(): + self.azimuthal = integrated_result.azimuthal + else: + self.azimuthal = None + self.propagate_uncertainties = (integrated_result.sigma is not None) + self.output = backup + def warmup(self, sync=False): """ Process a dummy image to ensure everything is initialized @@ -539,9 +582,7 @@ def warmup(self, sync=False): :param sync: wait for processing to be finished """ - t = threading.Thread(target=self.process, - name="init2d", - args=(numpy.zeros(self.shape, dtype=numpy.float32),)) + t = threading.Thread(target=self._warmup, name="_warmup") t.start() if sync: t.join()