Merge pull request #7 from LCOGT/rextract

Adding Plotting Functions for Re-extraction

banzai_floyds_ui/gui/utils/file_utils.py

@@ -0,0 +1,43 @@
+from astropy.io import fits
+from django.conf import settings
+import asyncio
+import httpx
+import io
+import requests
+
+
+async def fetch(url, params, headers):
+    async with httpx.AsyncClient() as client:
+        response = await client.get(url, params=params, headers=headers)
+    return response
+
+
+async def fetch_all(archive_header, request_params):
+    tasks = [fetch(settings.ARCHIVE_URL, params, archive_header) for params in request_params]
+    return await asyncio.gather(*tasks)
+
+
+def download_frame(headers, url=f'{settings.ARCHIVE_URL}', params=None, list_endpoint=False):
+    response = requests.get(url, params=params, headers=headers)
+    response.raise_for_status()
+
+    buffer = io.BytesIO()
+    if list_endpoint:
+        data_url = response.json()['results'][0]['url']
+    else:
+        data_url = response.json()['url']
+    data = requests.get(data_url).content
+    buffer.write(data)
+    buffer.seek(0)
+
+    hdu = fits.open(buffer)
+    return hdu
+
+
+def get_related_frame(frame_id, archive_header, related_frame_key):
+    # Get the related frame from the archive that matches related_frame_key in the header.
+    response = requests.get(f'{settings.ARCHIVE_URL}{frame_id}/headers', headers=archive_header)
+    response.raise_for_status()
+    related_frame_filename = response.json()['data'][related_frame_key]
+    params = {'basename_exact': related_frame_filename}
+    return download_frame(archive_header, params=params, list_endpoint=True), related_frame_filename

banzai_floyds_ui/gui/utils/header_utils.py

@@ -0,0 +1,11 @@
+import numpy as np
+
+
+def header_to_polynomial(header, key_pattern, order, max_coef=10):
+    domain = header[f'O{order}{key_pattern}DM0'], header[f'O{order}{key_pattern}DM1']
+    coef = []
+    for i in range(max_coef):
+        key = f'O{order}{key_pattern}{i:02}'
+        if key in header:
+            coef.append(header[key])
+    return np.polynomial.Legendre(coef, domain=domain)

banzai_floyds_ui/gui/utils/plot_utils.py

@@ -0,0 +1,57 @@
+import numpy as np
+
+
+def xy_to_svg_path(x, y):
+    # We don't want a Z at the end because we're not closing the path
+    return 'M ' + ' L '.join(f'{i},{j}' for i, j in zip(x, y))
+
+
+def unfilled_histogram(x, y, color, name=None, legend=None, axis=''):
+    # I didn't like how the plotly histogram looked so I wrote my own
+    x_avgs = (x[1:] + x[:-1]) / 2.0
+    x_lows = np.hstack([x[0] + x[0] - x_avgs[0], x_avgs])
+    x_highs = np.hstack([x_avgs, x[-1] + x[-1] - x_avgs[-1]])
+    x_plot = []
+    y_plot = []
+    for x_low, x_center, x_high, y_center in zip(x_lows, x, x_highs, y):
+        x_plot.append(x_low)
+        x_plot.append(x_center)
+        x_plot.append(x_high)
+        # Make the flat top at -1, 0, +1 of x
+        for _ in range(3):
+            y_plot.append(y_center / np.max(y))
+    show_legend = name is not None
+    if axis == 1:
+        axis = ''
+    return dict(type='scatter', x=x_plot, y=y_plot, mode='lines', line={'color': color}, hoverinfo='skip',
+                name=name, showlegend=show_legend, legend=legend, xaxis=f'x{axis}', yaxis=f'y{axis}')
+
+
+def json_to_polynomial(poly_info):
+    return np.polynomial.legendre.Legendre(coef=poly_info['coef'], domain=poly_info['domain'])
+
+
+EXTRACTION_REGION_LINE_ORDER = ['center', 'extract_lower', 'extract_upper', 'bkg_left_inner', 'bkg_right_inner',
+                                'bkg_left_outer', 'bkg_right_outer']
+
+
+def extraction_region_traces(order_polynomial, center_polynomial, width_polynomial,
+                             wavelengths_polynomial, extract_lower, extract_upper,
+                             bkg_left_inner, bkg_right_inner, bkg_left_outer, bkg_right_outer,
+                             center_delta=0.0):
+    x = np.arange(wavelengths_polynomial.domain[0], wavelengths_polynomial.domain[1] + 1)
+    wavelengths = wavelengths_polynomial(x)
+    extract_sigma = width_polynomial(wavelengths)
+    extract_center = center_polynomial(wavelengths)
+
+    extract_center += order_polynomial(x)
+    extract_center += center_delta
+    extract_high = extract_center + extract_upper * extract_sigma
+    extract_low = extract_center - extract_lower * extract_sigma
+    background_upper_start = extract_center + bkg_right_inner * extract_sigma
+    background_lower_start = extract_center - bkg_left_inner * extract_sigma
+    # Let's start with 10 sigma for now as the edge rather than using the whole slit
+    background_lower_end = extract_center - bkg_left_outer * extract_sigma
+    background_upper_end = extract_center + bkg_right_outer * extract_sigma
+    return x, [extract_center, extract_low, extract_high, background_lower_start, background_upper_start,
+               background_lower_end, background_upper_end]

banzai_floyds_ui/tests/mock_archive.py

@@ -0,0 +1,83 @@
+from flask import Flask, request, jsonify, send_file, url_for
+from glob import glob
+import io
+import os
+from astropy.io import fits
+
+file_list = glob('**/*.fits*', recursive=True)
+
+app = Flask(__name__)
+
+frames = {}
+for i, filename in enumerate(file_list):
+    hdu = fits.open(filename)
+    if 'SCI' in hdu:
+        hdu_index = 'SCI'
+    else:
+        hdu_index = 0
+    try:
+        frames[i] = {'id': i,
+                     'filename': os.path.basename(filename),
+                     'INSTRUME': hdu[hdu_index].header['INSTRUME'],
+                     'SITEID': hdu[hdu_index].header['INSTRUME'],
+                     'filepath': os.path.dirname(filename),}
+    except KeyError:
+        continue
+
+
+@app.route('/', methods=['GET'])
+@app.route('/<int:frame_id>/', methods=['GET'])
+def get_frame_by_params(frame_id=None):
+    if frame_id is not None:
+        if frame_id in frames:
+            frame = frames[frame_id]
+            frame['url'] = url_for('download_frame', frame_id=frame_id, _external=True)
+            return jsonify(frame), 201
+        else:
+            return jsonify({"error": "Frame not found"}), 404
+    instrument = request.args.get('instrument_id')
+    basename = request.args.get('basename')
+    basename_exact = request.args.get('basename_exact')
+    response_frames = []
+    for frame_id, frame in frames.items():
+        frame_criteria = frame['INSTRUME'] == instrument
+        if basename is not None:
+            frame_criteria = frame_criteria and basename in frame['filename']
+        if basename_exact is not None:
+            frame_criteria = basename_exact == frame['filename']
+        if frame_criteria:
+            frame['url'] = url_for('download_frame', frame_id=frame_id, _external=True)
+            response_frames.append(frame)
+
+    return jsonify({'results': response_frames}), 201
+
+
+@app.route('/download/<int:frame_id>', methods=['GET'])
+def download_frame(frame_id):
+
+    if frame_id in frames:
+        with open(os.path.join(frames[frame_id]['filepath'], frames[frame_id]['filename']), 'rb') as f:
+            return send_file(
+                io.BytesIO(f.read()),
+                download_name=frames[frame_id]['filename'],
+                mimetype='application/fits'
+            ), 200
+    else:
+        return jsonify({"error": "Frame not found"}), 404
+
+
+@app.route('/<int:frame_id>/headers', methods=['GET'])
+def get_header(frame_id):
+    if frame_id in frames:
+        hdu = fits.open(os.path.join(frames[frame_id]['filepath'], frames[frame_id]['filename']))
+        if 'SCI' in hdu:
+            hdu_index = 'SCI'
+        else:
+            hdu_index = 0
+        return jsonify({'data': dict(hdu[hdu_index].header)}), 200
+    else:
+        return jsonify({"error": "Frame not found"}), 404
+
+
+if __name__ == '__main__':
+    app.run(debug=True)

pyproject.toml

@@ -7,16 +7,16 @@ include = ["banzai_floyds_ui*"]
 
 [tool.setuptools.package-data]
 "banzai_floyds_ui" = ["static/*"]
-"banzai_floyds_ui.gui" = ["templates/*"]
+"banzai_floyds_ui.gui" = ["templates/*", "data/*"]
 
 [project]
 dynamic = ["version"]
 name = "banzai_floyds_ui"
 dependencies = [
   "django",
-  "dash",
+  "dash>=2.17",
   "plotly",
-  "django-plotly-dash",
+  "django-plotly-dash>=2.3.2",
   "astropy",
   "lcogt_logging",
   "gunicorn",
@@ -29,6 +29,11 @@ dependencies = [
   "django_redis",
   "whitenoise",
   "httpx",
-  "banzai_floyds @ git+https://github.com/lcogt/banzai-floyds@output-dataproduct-frameid",
+  "banzai_floyds @ git+https://github.com/lcogt/banzai-floyds@reextract",
   "scipy"
 ]
+
+[project.optional-dependencies]
+test = [
+    "flask"
+]