From 81c0c59837b464ca4deffbecf523c7e8a264627a Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 11 Dec 2024 10:46:09 -0500 Subject: [PATCH 1/3] Prefer same block/proposal for calibrations. Only use public calibrations. Use calibrations with the same slit width (i.e. same slit mask) for processing. --- CHANGES.md | 8 ++++++++ banzai_floyds/frames.py | 4 ++++ banzai_floyds/fringe.py | 1 + banzai_floyds/settings.py | 5 +++++ helm-chart/banzai-floyds/templates/cronjob.yaml | 4 ++++ helm-chart/banzai-floyds/templates/listener.yaml | 4 ++++ setup.cfg | 2 +- 7 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 86f7549..1fc0e89 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +0.12.0 (2024-12-11) +------------------- +- We now prefer using calibrations, specifically arcs, from the same block or the same proposal +- We only use public calibrations now +- If a block is still going, we delay the processing in case there is a calibration taken at + at the end of the block that we can use for processing +- We now only use arcs and flats taken with the same slit width as the science data + 0.11.2 (2024-11-18) ------------------- - Simplified the fitting for refining the peak centers. We no longer try to fit them all simultaneously diff --git a/banzai_floyds/frames.py b/banzai_floyds/frames.py index 35b3c3d..0bf735a 100644 --- a/banzai_floyds/frames.py +++ b/banzai_floyds/frames.py @@ -236,6 +236,10 @@ def elevation(self): def elevation(self, value): self.meta['HEIGHT'] = value + @property + def slit_width(self): + return self.meta['APERWID'] + class FLOYDSCalibrationFrame(LCOCalibrationFrame, FLOYDSObservationFrame): def __init__(self, hdu_list: list, file_path: str, frame_id: int = None, grouping_criteria: list = None, diff --git a/banzai_floyds/fringe.py b/banzai_floyds/fringe.py index 2f057fd..258a700 100644 --- a/banzai_floyds/fringe.py +++ b/banzai_floyds/fringe.py @@ -112,6 +112,7 @@ def do_stage(self, image): image.uncertainty[to_correct] /= fringe_correction[fringe_correction > 0.1] image.meta['L1FRNGOF'] = (fringe_offset, 'Fringe offset (pixels)') image.meta['L1STATFR'] = (1, 'Status flag for fringe frame correction') + image.public_date = datetime.now() return image diff --git a/banzai_floyds/settings.py b/banzai_floyds/settings.py index 5026990..142a309 100644 --- a/banzai_floyds/settings.py +++ b/banzai_floyds/settings.py @@ -52,3 +52,8 @@ CALIBRATION_IMAGE_TYPES = ['BIAS', 'DARK', 'SKYFLAT', 'BPM', 'LAMPFLAT', 'ARC', 'STANDARD'] CALIBRATION_LOOKBACK = {'LAMPFLAT': 2.5} + +CALIBRATION_SET_CRITERIA = {'SKYFLAT': [], 'LAMPFLAT': ['slit_width'], 'ARC': ['slit_width'], + 'FRINGE': ['slit_width']} + +OBSTYPES_TO_DELAY = ['STANDARD', 'SPECTRUM'] diff --git a/helm-chart/banzai-floyds/templates/cronjob.yaml b/helm-chart/banzai-floyds/templates/cronjob.yaml index f90254d..9d7d4f0 100644 --- a/helm-chart/banzai-floyds/templates/cronjob.yaml +++ b/helm-chart/banzai-floyds/templates/cronjob.yaml @@ -38,5 +38,9 @@ spec: - "--rlevel=91" - "--db-address=$(DB_ADDRESS)" - "--no-file-cache" + - "--delay-to-block-end" + - "--same-block-cals" + - "--check-public-cals" + - "--prefer-same-proposal" restartPolicy: Never {{- end }} diff --git a/helm-chart/banzai-floyds/templates/listener.yaml b/helm-chart/banzai-floyds/templates/listener.yaml index 841c1e9..37067ea 100644 --- a/helm-chart/banzai-floyds/templates/listener.yaml +++ b/helm-chart/banzai-floyds/templates/listener.yaml @@ -38,6 +38,10 @@ spec: - "--broker-url=$(FITS_BROKER)" - "--queue-name=$(QUEUE_NAME)" - "--no-file-cache" + - "--delay-to-block-end" + - "--same-block-cals" + - "--check-public-cals" + - "--prefer-same-proposal" env: {{- include "banzaiFloyds.Env" . | nindent 12}} resources: diff --git a/setup.cfg b/setup.cfg index 75382e4..3b6cc5c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ setup_requires = psycopg2-binary install_requires = astropy >= 4.3 - banzai @ git+https://github.com/lcogt/banzai.git@1.19.1 + banzai @ git+https://github.com/lcogt/banzai.git@cals-by-block.git matplotlib [options.extras_require] From 3e7271728c971cfc6d8407d8559dec7e08bb4270 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Mon, 16 Dec 2024 10:08:56 -0500 Subject: [PATCH 2/3] Fixes to install and changelog. --- CHANGES.md | 3 +-- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1fc0e89..4358687 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,6 @@ 0.12.0 (2024-12-11) ------------------- -- We now prefer using calibrations, specifically arcs, from the same block or the same proposal -- We only use public calibrations now +- We now have prefer calibrations in the following order: same block, same proposal, any public calibration. - If a block is still going, we delay the processing in case there is a calibration taken at at the end of the block that we can use for processing - We now only use arcs and flats taken with the same slit width as the science data diff --git a/setup.cfg b/setup.cfg index 3b6cc5c..12547e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ setup_requires = psycopg2-binary install_requires = astropy >= 4.3 - banzai @ git+https://github.com/lcogt/banzai.git@cals-by-block.git + banzai @ git+https://github.com/lcogt/banzai.git@cals-by-block matplotlib [options.extras_require] From cba11b48dead7e511e2e3213a1a36bc7250e7ed5 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Thu, 6 Feb 2025 16:48:25 -0500 Subject: [PATCH 3/3] Added upstream banzai tagged version. --- Dockerfile | 2 +- helm-chart/banzai-floyds-e2e/templates/deployment.yaml | 3 +++ helm-chart/banzai-floyds/templates/listener.yaml | 4 ++-- setup.cfg | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 79dfe70..ca79fac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/lcogt/banzai:1.19.1 +FROM ghcr.io/lcogt/banzai:1.20.1 USER root diff --git a/helm-chart/banzai-floyds-e2e/templates/deployment.yaml b/helm-chart/banzai-floyds-e2e/templates/deployment.yaml index 5ed7707..8b242b1 100644 --- a/helm-chart/banzai-floyds-e2e/templates/deployment.yaml +++ b/helm-chart/banzai-floyds-e2e/templates/deployment.yaml @@ -155,6 +155,9 @@ spec: - "--broker-url=localhost" - "--log-level=debug" - "--rlevel=91" + - "--prefer-same-block-cals" + - "--check-public-cals" + - "--prefer-same-proposal-cals" resources: requests: cpu: "0.1" diff --git a/helm-chart/banzai-floyds/templates/listener.yaml b/helm-chart/banzai-floyds/templates/listener.yaml index 37067ea..e16e6e0 100644 --- a/helm-chart/banzai-floyds/templates/listener.yaml +++ b/helm-chart/banzai-floyds/templates/listener.yaml @@ -39,9 +39,9 @@ spec: - "--queue-name=$(QUEUE_NAME)" - "--no-file-cache" - "--delay-to-block-end" - - "--same-block-cals" + - "--prefer-same-block-cals" - "--check-public-cals" - - "--prefer-same-proposal" + - "--prefer-same-proposal-cals" env: {{- include "banzaiFloyds.Env" . | nindent 12}} resources: diff --git a/setup.cfg b/setup.cfg index 12547e9..eae5a92 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ setup_requires = psycopg2-binary install_requires = astropy >= 4.3 - banzai @ git+https://github.com/lcogt/banzai.git@cals-by-block + banzai @ git+https://github.com/lcogt/banzai.git@1.20.1 matplotlib [options.extras_require]