From 35993be7872125fc56039abdf1197cca6552fc7c Mon Sep 17 00:00:00 2001 From: Stelios Voutsinas Date: Mon, 23 Sep 2024 16:01:47 -0700 Subject: [PATCH 1/9] Added Docs build to ci workflow Undo changes to ci.yaml --- .pre-commit-config.yaml | 4 +- Dockerfile | 2 +- Makefile | 2 +- README.md | 18 +- pyproject.toml | 20 +- requirements/dev.txt | 116 +++++ requirements/main.in | 9 + requirements/main.txt | 214 +++++++++ requirements/tox.txt | 41 ++ src/{vosiav2 => sia}/__init__.py | 4 +- src/sia/config.py | 55 +++ src/sia/constants.py | 15 + src/sia/data/dp02.yaml | 53 +++ .../handlers => sia/dependencies}/__init__.py | 0 src/sia/dependencies/availability.py | 17 + src/sia/dependencies/context.py | 131 ++++++ .../dependencies/labeled_butler_factory.py | 40 ++ src/sia/dependencies/query_params.py | 64 +++ src/sia/dependencies/token.py | 30 ++ src/sia/exceptions.py | 270 ++++++++++++ src/sia/factory.py | 164 +++++++ src/sia/handlers/__init__.py | 0 src/sia/handlers/external.py | 297 +++++++++++++ src/{vosiav2 => sia}/handlers/internal.py | 4 +- src/sia/main.py | 105 +++++ src/sia/middleware/__init__.py | 0 src/sia/middleware/ivoa.py | 140 ++++++ src/sia/models/__init__.py | 0 src/sia/models/butler_type.py | 10 + src/sia/models/common.py | 37 ++ src/sia/models/data_collections.py | 53 +++ .../models.py => sia/models/index.py} | 2 +- src/sia/models/sia_query_params.py | 411 ++++++++++++++++++ src/sia/services/__init__.py | 0 src/sia/services/availability.py | 111 +++++ src/sia/services/data_collections.py | 105 +++++ src/sia/services/response_handler.py | 103 +++++ src/sia/services/votable.py | 35 ++ src/sia/templates/availability.xml | 3 + src/sia/templates/capabilities.xml | 25 ++ src/sia/templates/votable_error.xml | 6 + src/sia/timer.py | 64 +++ src/vosiav2/config.py | 35 -- src/vosiav2/handlers/external.py | 52 --- src/vosiav2/main.py | 60 --- tests/conftest.py | 224 +++++++++- tests/data/config/ci_hsc_gen3.yaml | 58 +++ tests/data/config/dp02.yaml | 53 +++ tests/data/repo/butler.yaml | 14 + tests/data/repo/gen3.sqlite3 | Bin 0 -> 18653184 bytes tests/exceptions/__init__.py | 0 tests/exceptions/exceptions_test.py | 47 ++ tests/handlers/external/__init__.py | 0 tests/handlers/external/availability_test.py | 119 +++++ tests/handlers/external/capabilities_test.py | 41 ++ tests/handlers/external/external_test.py | 229 ++++++++++ tests/handlers/external_test.py | 22 - tests/handlers/internal_test.py | 4 +- tests/models/__init__.py | 0 tests/models/sia_params_test.py | 160 +++++++ tests/services/__init__.py | 0 tests/services/data_collection_test.py | 164 +++++++ tests/services/timer_test.py | 53 +++ tests/support/__init__.py | 0 tests/support/butler.py | 137 ++++++ tests/support/constants.py | 7 + tests/support/validators.py | 113 +++++ tests/templates/availability.xml | 1 + tests/templates/capabilities.xml | 25 ++ tests/templates/expected_votable.xml | 36 ++ tox.ini | 6 +- 71 files changed, 4240 insertions(+), 200 deletions(-) create mode 100644 requirements/dev.txt create mode 100644 requirements/main.txt create mode 100644 requirements/tox.txt rename src/{vosiav2 => sia}/__init__.py (80%) create mode 100644 src/sia/config.py create mode 100644 src/sia/constants.py create mode 100644 src/sia/data/dp02.yaml rename src/{vosiav2/handlers => sia/dependencies}/__init__.py (100%) create mode 100644 src/sia/dependencies/availability.py create mode 100644 src/sia/dependencies/context.py create mode 100644 src/sia/dependencies/labeled_butler_factory.py create mode 100644 src/sia/dependencies/query_params.py create mode 100644 src/sia/dependencies/token.py create mode 100644 src/sia/exceptions.py create mode 100644 src/sia/factory.py create mode 100644 src/sia/handlers/__init__.py create mode 100644 src/sia/handlers/external.py rename src/{vosiav2 => sia}/handlers/internal.py (93%) create mode 100644 src/sia/main.py create mode 100644 src/sia/middleware/__init__.py create mode 100644 src/sia/middleware/ivoa.py create mode 100644 src/sia/models/__init__.py create mode 100644 src/sia/models/butler_type.py create mode 100644 src/sia/models/common.py create mode 100644 src/sia/models/data_collections.py rename src/{vosiav2/models.py => sia/models/index.py} (92%) create mode 100644 src/sia/models/sia_query_params.py create mode 100644 src/sia/services/__init__.py create mode 100644 src/sia/services/availability.py create mode 100644 src/sia/services/data_collections.py create mode 100644 src/sia/services/response_handler.py create mode 100644 src/sia/services/votable.py create mode 100644 src/sia/templates/availability.xml create mode 100644 src/sia/templates/capabilities.xml create mode 100644 src/sia/templates/votable_error.xml create mode 100644 src/sia/timer.py delete mode 100644 src/vosiav2/config.py delete mode 100644 src/vosiav2/handlers/external.py delete mode 100644 src/vosiav2/main.py create mode 100644 tests/data/config/ci_hsc_gen3.yaml create mode 100644 tests/data/config/dp02.yaml create mode 100644 tests/data/repo/butler.yaml create mode 100644 tests/data/repo/gen3.sqlite3 create mode 100644 tests/exceptions/__init__.py create mode 100644 tests/exceptions/exceptions_test.py create mode 100644 tests/handlers/external/__init__.py create mode 100644 tests/handlers/external/availability_test.py create mode 100644 tests/handlers/external/capabilities_test.py create mode 100644 tests/handlers/external/external_test.py delete mode 100644 tests/handlers/external_test.py create mode 100644 tests/models/__init__.py create mode 100644 tests/models/sia_params_test.py create mode 100644 tests/services/__init__.py create mode 100644 tests/services/data_collection_test.py create mode 100644 tests/services/timer_test.py create mode 100644 tests/support/__init__.py create mode 100644 tests/support/butler.py create mode 100644 tests/support/constants.py create mode 100644 tests/support/validators.py create mode 100644 tests/templates/availability.xml create mode 100644 tests/templates/capabilities.xml create mode 100644 tests/templates/expected_votable.xml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d302149..239015a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-toml - id: check-yaml - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.5 + rev: v0.6.9 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/Dockerfile b/Dockerfile index 8073cb0..862e7d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,4 +65,4 @@ USER appuser EXPOSE 8080 # Run the application. -CMD ["uvicorn", "vosiav2.main:app", "--host", "0.0.0.0", "--port", "8080"] +CMD ["uvicorn", "sia.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/Makefile b/Makefile index 07422a7..d41e907 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: help help: - @echo "Make targets for vo-siav2" + @echo "Make targets for sia" @echo "make init - Set up dev environment" @echo "make run - Start a local development instance" @echo "make update - Update pinned dependencies and run make init" diff --git a/README.md b/README.md index 1962e3f..dee0742 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,16 @@ -# vo-siav2 +# SIA -Rubin Observatory SIAV2 implementation over Butler -Learn more at https://vo-siav2.lsst.io +SIA is an implementation of the IVOA [Simple Image Access v2](https://www.ivoa.net/documents/SIA/20150610/PR-SIA-2.0-20150610.pdf) protocol as a [FastAPI](https://fastapi.tiangolo.com/) web service, designed to be deployed as part of the Rubin Science Platform. + +The default configuration is designed to use the [dax_obscore](https://github.com/lsst-dm/dax_obscore) package and interact with a [Butler](https://github.com/lsst/daf_butler) repository to find images that match a certain criteria. + +While the application has been designed with consideration to potential future use with other middleware packages & query engines, the current release is targeted to the specific Butler-backed use case for the RSP. + +Queries results are streamed as VOTable responses to the user, and in the current release this is the only format supported. + +The application expects as configuration the definition of what query_engine to use, and the associated data collections configuration. In the default case of using Remote Butler as the query engine, the application expects at least one data collection (with default=True). + +The system architecture & design considerations have been documented in https://github.com/lsst-sqre/sqr-095. + +See [CHANGELOG.md](https://github.com/lsst-sqre/sia/blob/main/CHANGELOG.md) for the change history of sia. -vo-siav2 is developed with [FastAPI](https://fastapi.tiangolo.com) and [Safir](https://safir.lsst.io). diff --git a/pyproject.toml b/pyproject.toml index 0beab2e..fcb6bc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] # https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ -name = "vo-siav2" -description = "Rubin Observatory SIAV2 implementation over Butler" +name = "sia" +description = "Rubin Observatory SIA implementation over Butler" license = { file = "LICENSE" } readme = "README.md" keywords = ["rubin", "lsst"] @@ -23,11 +23,11 @@ dependencies = [] dynamic = ["version"] [project.scripts] -vo-siav2 = "vosiav2.cli:main" +sia = "sia.cli:main" [project.urls] -Homepage = "https://vo-siav2.lsst.io" -Source = "https://github.com/lsst-sqre/vo-siav2" +Homepage = "https://sia.lsst.io" +Source = "https://github.com/lsst-sqre/sia" [build-system] requires = ["setuptools>=61", "wheel", "setuptools_scm[toml]>=6.2"] @@ -35,10 +35,13 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] +[tool.setuptools.package-data] +"sia" = ["templates/*.xml"] + [tool.coverage.run] parallel = true branch = true -source = ["vosiav2"] +source = ["sia"] [tool.coverage.paths] source = ["src", ".tox/*/site-packages"] @@ -62,7 +65,7 @@ disallow_untyped_defs = true disallow_incomplete_defs = true ignore_missing_imports = true local_partial_types = true -plugins = ["pydantic.mypy"] +plugins = ["pydantic.mypy","pydantic_xml.mypy"] no_implicit_reexport = true show_error_codes = true strict_equality = true @@ -87,6 +90,7 @@ asyncio_mode = "strict" # with complex data structures rather than only the assert message) in files # listed in python_files. python_files = ["tests/*.py", "tests/*/*.py"] +asyncio_default_fixture_loop_scope = "function" # Use the generic Ruff configuration in ruff.toml and extend it with only # project-specific settings. Add a [tool.ruff.lint.extend-per-file-ignores] @@ -95,7 +99,7 @@ python_files = ["tests/*.py", "tests/*/*.py"] extend = "ruff-shared.toml" [tool.ruff.lint.isort] -known-first-party = ["vosiav2", "tests"] +known-first-party = ["sia", "tests"] split-on-trailing-comma = false [tool.scriv] diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 0000000..9b1e377 --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,116 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file requirements/dev.txt requirements/dev.in +annotated-types==0.7.0 + # via + # -c requirements/main.txt + # pydantic +anyio==4.6.0 + # via + # -c requirements/main.txt + # httpx +asgi-lifespan==2.1.0 + # via -r requirements/dev.in +attrs==24.2.0 + # via scriv +certifi==2024.8.30 + # via + # -c requirements/main.txt + # httpcore + # httpx + # requests +charset-normalizer==3.3.2 + # via + # -c requirements/main.txt + # requests +click==8.1.7 + # via + # -c requirements/main.txt + # click-log + # scriv +click-log==0.4.0 + # via scriv +coverage==7.6.1 + # via + # -r requirements/dev.in + # pytest-cov +h11==0.14.0 + # via + # -c requirements/main.txt + # httpcore +httpcore==1.0.6 + # via + # -c requirements/main.txt + # httpx +httpx==0.27.2 + # via + # -c requirements/main.txt + # -r requirements/dev.in +idna==3.10 + # via + # -c requirements/main.txt + # anyio + # httpx + # requests +iniconfig==2.0.0 + # via pytest +jinja2==3.1.4 + # via + # -c requirements/main.txt + # scriv +markdown-it-py==3.0.0 + # via scriv +markupsafe==3.0.0 + # via + # -c requirements/main.txt + # jinja2 +mdurl==0.1.2 + # via markdown-it-py +mypy==1.11.2 + # via -r requirements/dev.in +mypy-extensions==1.0.0 + # via mypy +packaging==24.1 + # via + # -c requirements/main.txt + # pytest +pluggy==1.5.0 + # via pytest +pydantic==2.9.2 + # via + # -c requirements/main.txt + # -r requirements/dev.in +pydantic-core==2.23.4 + # via + # -c requirements/main.txt + # pydantic +pytest==8.3.3 + # via + # -r requirements/dev.in + # pytest-asyncio + # pytest-cov +pytest-asyncio==0.24.0 + # via -r requirements/dev.in +pytest-cov==5.0.0 + # via -r requirements/dev.in +requests==2.32.3 + # via + # -c requirements/main.txt + # scriv +scriv==1.5.1 + # via -r requirements/dev.in +sniffio==1.3.1 + # via + # -c requirements/main.txt + # anyio + # asgi-lifespan + # httpx +typing-extensions==4.12.2 + # via + # -c requirements/main.txt + # mypy + # pydantic + # pydantic-core +urllib3==2.2.3 + # via + # -c requirements/main.txt + # requests diff --git a/requirements/main.in b/requirements/main.in index 985d2fe..b3cf854 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -11,8 +11,17 @@ fastapi starlette uvicorn[standard] +python-multipart # Other dependencies. +requests +jinja2 pydantic pydantic-settings safir>=5 +numpy +astropy +vo-models==0.3.1 +defusedxml +lsst-daf-butler[remote] +lsst-dax-obscore @ git+https://github.com/lsst-dm/dax_obscore@main diff --git a/requirements/main.txt b/requirements/main.txt new file mode 100644 index 0000000..e022163 --- /dev/null +++ b/requirements/main.txt @@ -0,0 +1,214 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file requirements/main.txt requirements/main.in +annotated-types==0.7.0 + # via pydantic +anyio==4.6.0 + # via + # httpx + # starlette + # watchfiles +astropy==6.1.4 + # via + # -r requirements/main.in + # lsst-daf-butler + # lsst-felis + # lsst-resources + # lsst-utils +astropy-iers-data==0.2024.10.7.0.32.46 + # via astropy +certifi==2024.8.30 + # via + # httpcore + # httpx + # requests +cffi==1.17.1 + # via cryptography +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # lsst-daf-butler + # lsst-dax-obscore + # lsst-felis + # safir + # uvicorn +cryptography==43.0.1 + # via + # pyjwt + # safir +defusedxml==0.7.1 + # via + # -r requirements/main.in + # lsst-resources +deprecated==1.2.14 + # via + # lsst-daf-butler + # lsst-daf-relation + # lsst-utils +fastapi==0.115.0 + # via + # -r requirements/main.in + # safir +gidgethub==5.3.0 + # via safir +greenlet==3.1.1 + # via sqlalchemy +h11==0.14.0 + # via + # httpcore + # uvicorn +hpgeom==1.4.0 + # via lsst-sphgeom +httpcore==1.0.6 + # via httpx +httptools==0.6.1 + # via uvicorn +httpx==0.27.2 + # via + # lsst-daf-butler + # safir +idna==3.10 + # via + # anyio + # httpx + # requests +jinja2==3.1.4 + # via -r requirements/main.in +lsst-daf-butler==27.2024.4000 + # via + # -r requirements/main.in + # lsst-dax-obscore +lsst-daf-relation==27.2024.4000 + # via lsst-daf-butler +lsst-dax-obscore @ git+https://github.com/lsst-dm/dax_obscore@5919c78c29630b1c9ed7544a1e075d8cc7ebda22 + # via -r requirements/main.in +lsst-felis==27.2024.4000 + # via lsst-dax-obscore +lsst-resources==27.2024.4000 + # via + # lsst-daf-butler + # lsst-dax-obscore +lsst-sphgeom==27.2024.3700 + # via + # lsst-daf-butler + # lsst-dax-obscore +lsst-utils==27.2024.4000 + # via + # lsst-daf-butler + # lsst-daf-relation + # lsst-dax-obscore + # lsst-felis + # lsst-resources +lxml==5.3.0 + # via pydantic-xml +markupsafe==3.0.0 + # via jinja2 +numpy==2.1.2 + # via + # -r requirements/main.in + # astropy + # hpgeom + # lsst-sphgeom + # lsst-utils + # pyarrow + # pyerfa +packaging==24.1 + # via astropy +psutil==6.0.0 + # via lsst-utils +pyarrow==17.0.0 + # via + # lsst-daf-butler + # lsst-dax-obscore +pycparser==2.22 + # via cffi +pydantic==2.9.2 + # via + # -r requirements/main.in + # fastapi + # lsst-daf-butler + # lsst-daf-relation + # lsst-felis + # pydantic-settings + # pydantic-xml + # safir +pydantic-core==2.23.4 + # via + # pydantic + # pydantic-xml + # safir +pydantic-settings==2.5.2 + # via -r requirements/main.in +pydantic-xml==2.13.1 + # via vo-models +pyerfa==2.0.1.4 + # via astropy +pyjwt==2.9.0 + # via gidgethub +python-dotenv==1.0.1 + # via + # pydantic-settings + # uvicorn +python-multipart==0.0.12 + # via -r requirements/main.in +pyyaml==6.0.2 + # via + # astropy + # lsst-daf-butler + # lsst-dax-obscore + # lsst-felis + # lsst-utils + # uvicorn +requests==2.32.3 + # via + # -r requirements/main.in + # lsst-resources +safir==6.4.0 + # via -r requirements/main.in +safir-logging==6.4.0 + # via safir +sniffio==1.3.1 + # via + # anyio + # httpx +sqlalchemy==2.0.35 + # via + # lsst-daf-butler + # lsst-daf-relation + # lsst-dax-obscore + # lsst-felis +starlette==0.38.6 + # via + # -r requirements/main.in + # fastapi + # safir +structlog==24.4.0 + # via + # safir + # safir-logging +threadpoolctl==3.5.0 + # via lsst-utils +typing-extensions==4.12.2 + # via + # fastapi + # pydantic + # pydantic-core + # sqlalchemy +uritemplate==4.1.1 + # via gidgethub +urllib3==2.2.3 + # via + # lsst-resources + # requests +uvicorn==0.31.0 + # via -r requirements/main.in +uvloop==0.20.0 + # via uvicorn +vo-models==0.3.1 + # via -r requirements/main.in +watchfiles==0.24.0 + # via uvicorn +websockets==13.1 + # via uvicorn +wrapt==1.16.0 + # via deprecated diff --git a/requirements/tox.txt b/requirements/tox.txt new file mode 100644 index 0000000..5502abd --- /dev/null +++ b/requirements/tox.txt @@ -0,0 +1,41 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file requirements/tox.txt requirements/tox.in +cachetools==5.5.0 + # via tox +chardet==5.2.0 + # via tox +colorama==0.4.6 + # via tox +distlib==0.3.8 + # via virtualenv +filelock==3.16.1 + # via + # tox + # virtualenv +packaging==24.1 + # via + # -c requirements/dev.txt + # -c requirements/main.txt + # pyproject-api + # tox + # tox-uv +platformdirs==4.3.6 + # via + # tox + # virtualenv +pluggy==1.5.0 + # via + # -c requirements/dev.txt + # tox +pyproject-api==1.8.0 + # via tox +tox==4.21.2 + # via + # -r requirements/tox.in + # tox-uv +tox-uv==1.13.0 + # via -r requirements/tox.in +uv==0.4.18 + # via tox-uv +virtualenv==20.26.6 + # via tox diff --git a/src/vosiav2/__init__.py b/src/sia/__init__.py similarity index 80% rename from src/vosiav2/__init__.py rename to src/sia/__init__.py index 12627dd..928da73 100644 --- a/src/vosiav2/__init__.py +++ b/src/sia/__init__.py @@ -1,4 +1,4 @@ -"""The vo-siav2 service.""" +"""The SIA service.""" __all__ = ["__version__"] @@ -8,7 +8,7 @@ """The application version string (PEP 440 / SemVer compatible).""" try: - __version__ = version("vo-siav2") + __version__ = version("sia") except PackageNotFoundError: # package is not installed __version__ = "0.0.0" diff --git a/src/sia/config.py b/src/sia/config.py new file mode 100644 index 0000000..8b6ee83 --- /dev/null +++ b/src/sia/config.py @@ -0,0 +1,55 @@ +"""Configuration definition.""" + +from __future__ import annotations + +from typing import Annotated + +from pydantic import Field, HttpUrl +from pydantic_settings import BaseSettings, SettingsConfigDict +from safir.logging import LogLevel, Profile + +from .models.butler_type import ButlerType +from .models.data_collections import ButlerDataCollection + +__all__ = ["Config", "config"] + + +class Config(BaseSettings): + """Configuration for sia.""" + + name: str = Field("sia", title="Name of application") + """Name of application.""" + + path_prefix: str = Field("/api/sia", title="URL prefix for application") + """URL prefix for application.""" + + profile: Profile = Field( + Profile.development, title="Application logging profile" + ) + """Application logging profile.""" + + log_level: LogLevel = Field( + LogLevel.INFO, title="Log level of the application's logger" + ) + """Log level of the application's logger.""" + + model_config = SettingsConfigDict(env_prefix="SIA_", case_sensitive=False) + """Configuration for the model settings.""" + + butler_type: ButlerType = ButlerType.REMOTE + """Configuration for the butler type.""" + + butler_data_collections: Annotated[ + list[ButlerDataCollection], + Field(title="Data collections"), + ] = [] + """Configuration for the data collections.""" + + slack_webhook: Annotated[ + HttpUrl | None, Field(title="Slack webhook for exception reporting") + ] = None + """Slack webhook for exception reporting.""" + + +config = Config() +"""Configuration instance for sia.""" diff --git a/src/sia/constants.py b/src/sia/constants.py new file mode 100644 index 0000000..b211204 --- /dev/null +++ b/src/sia/constants.py @@ -0,0 +1,15 @@ +"""Constants for the SIA service.""" + +__all__ = ["RESPONSEFORMATS", "RESULT_NAME", "SINGLE_PARAMS"] + +RESPONSEFORMATS = {"votable", "application/x-votable"} +"""List of supported response formats for the SIA service.""" + +RESULT_NAME = "result" +"""The name of the result file.""" + +SINGLE_PARAMS = { + "maxrec", + "responseformat", +} +"""Parameters that should be treated as single values.""" diff --git a/src/sia/data/dp02.yaml b/src/sia/data/dp02.yaml new file mode 100644 index 0000000..407a15e --- /dev/null +++ b/src/sia/data/dp02.yaml @@ -0,0 +1,53 @@ +facility_name: Rubin-LSST +obs_collection: LSST.DP02 +collections: ["2.2i/runs/DP0.2"] +use_butler_uri: false +dataset_types: + raw: + dataproduct_type: image + dataproduct_subtype: lsst.raw + calib_level: 1 + obs_id_fmt: "{records[exposure].obs_id}-{records[detector].full_name}" + o_ucd: phot.count + access_format: image/fits + datalink_url_fmt: "https://data.lsst.cloud/api/datalink/links?ID=butler%3A//dp02/{id}" + calexp: + dataproduct_type: image + dataproduct_subtype: lsst.calexp + calib_level: 2 + obs_id_fmt: "{records[visit].name}-{records[detector].full_name}" + o_ucd: phot.count + access_format: image/fits + datalink_url_fmt: "https://data.lsst.cloud/api/datalink/links?ID=butler%3A//dp02/{id}" + deepCoadd_calexp: + dataproduct_type: image + dataproduct_subtype: lsst.deepCoadd_calexp + calib_level: 3 + obs_id_fmt: "{skymap}-{tract}-{patch}" + o_ucd: phot.count + access_format: image/fits + datalink_url_fmt: "https://data.lsst.cloud/api/datalink/links?ID=butler%3A//dp02/{id}" + goodSeeingCoadd: + dataproduct_type: image + dataproduct_subtype: lsst.goodSeeingCoadd + calib_level: 3 + obs_id_fmt: "{skymap}-{tract}-{patch}" + o_ucd: phot.count + access_format: image/fits + datalink_url_fmt: "https://data.lsst.cloud/api/datalink/links?ID=butler%3A//dp02/{id}" + goodSeeingDiff_differenceExp: + dataproduct_type: image + dataproduct_subtype: lsst.goodSeeingDiff_differenceExp + calib_level: 3 + obs_id_fmt: "{records[visit].name}-{records[detector].full_name}" + o_ucd: phot.count + access_format: image/fits + datalink_url_fmt: "https://data.lsst.cloud/api/datalink/links?ID=butler%3A//dp02/{id}" + +spectral_ranges: + "u": [330.0e-9, 400.0e-9] + "g": [402.0e-9, 552.0e-9] + "r": [552.0e-9, 691.0e-9] + "i": [691.0e-9, 818.0e-9] + "z": [818.0e-9, 922.0e-9] + "y": [970.0e-9, 1060.0e-9] \ No newline at end of file diff --git a/src/vosiav2/handlers/__init__.py b/src/sia/dependencies/__init__.py similarity index 100% rename from src/vosiav2/handlers/__init__.py rename to src/sia/dependencies/__init__.py diff --git a/src/sia/dependencies/availability.py b/src/sia/dependencies/availability.py new file mode 100644 index 0000000..10462db --- /dev/null +++ b/src/sia/dependencies/availability.py @@ -0,0 +1,17 @@ +"""Dependency for the availability service.""" + +from vo_models.vosi.availability import Availability + +from ..config import config +from ..services.availability import AvailabilityService + + +async def get_availability_dependency() -> Availability: + """Return the availability of the service. + + Returns + ------- + Availability + The availability of the service. + """ + return await AvailabilityService(config=config).get_availability() diff --git a/src/sia/dependencies/context.py b/src/sia/dependencies/context.py new file mode 100644 index 0000000..ac923e9 --- /dev/null +++ b/src/sia/dependencies/context.py @@ -0,0 +1,131 @@ +"""Request context dependency for FastAPI. + +This dependency gathers a variety of information into a single object for the +convenience of writing request handlers. It also provides a place to store a +`structlog.BoundLogger` that can gather additional context during processing, +including from dependencies. +""" + +from dataclasses import dataclass +from typing import Annotated, Any + +from fastapi import Depends, Request +from safir.dependencies.logger import logger_dependency +from structlog.stdlib import BoundLogger + +from ..config import Config +from ..factory import Factory, ProcessContext + +__all__ = [ + "ContextDependency", + "RequestContext", + "context_dependency", +] + + +@dataclass(slots=True) +class RequestContext: + """Holds the incoming request and its surrounding context. + + The primary reason for the existence of this class is to allow the + functions involved in request processing to repeated rebind the request + logger to include more information, without having to pass both the + request and the logger separately to every function. + """ + + request: Request + """The incoming request.""" + + config: Config + """SIA's configuration.""" + + logger: BoundLogger + """The request logger, rebound with discovered context.""" + + factory: Factory + """The component factory.""" + + def rebind_logger(self, **values: Any) -> None: + """Add the given values to the logging context. + + Parameters + ---------- + **values + Additional values that should be added to the logging context. + """ + self.logger = self.logger.bind(**values) + self.factory.set_logger(self.logger) + + +class ContextDependency: + """Provide a per-request context as a FastAPI dependency. + + Each request gets a `RequestContext`. To save overhead, the portions of + the context that are shared by all requests are collected into the single + process-global `~sia.factory.ProcessContext` and reused with each + request. + """ + + def __init__(self) -> None: + self._config: Config | None = None + self._process_context: ProcessContext | None = None + + async def __call__( + self, + *, + request: Request, + logger: Annotated[BoundLogger, Depends(logger_dependency)], + ) -> RequestContext: + """Create a per-request context and return it.""" + if not self._config or not self._process_context: + raise RuntimeError("ContextDependency not initialized") + + return RequestContext( + request=request, + config=self._config, + logger=logger, + factory=Factory( + process_context=self._process_context, logger=logger + ), + ) + + @property + def process_context(self) -> ProcessContext: + """The underlying process context, primarily for use in tests.""" + if not self._process_context: + raise RuntimeError("ContextDependency not initialized") + return self._process_context + + def create_factory(self, logger: BoundLogger) -> Factory: + """Create a factory for use outside a request context.""" + return Factory( + logger=logger, + process_context=self.process_context, + ) + + async def aclose(self) -> None: + """Clean up the per-process configuration.""" + if self._process_context: + await self._process_context.aclose() + self._config = None + self._process_context = None + + async def initialize( + self, + config: Config, + ) -> None: + """Initialize the process-wide shared context. + + Parameters + ---------- + config + SIA configuration. + """ + if self._process_context: + await self._process_context.aclose() + self._config = config + self._process_context = await ProcessContext.create() + + +context_dependency = ContextDependency() +"""The dependency that will return the per-request context.""" diff --git a/src/sia/dependencies/labeled_butler_factory.py b/src/sia/dependencies/labeled_butler_factory.py new file mode 100644 index 0000000..73409e7 --- /dev/null +++ b/src/sia/dependencies/labeled_butler_factory.py @@ -0,0 +1,40 @@ +"""Dependency class for creating a LabeledButlerFactory singleton.""" + +from lsst.daf.butler import LabeledButlerFactory + +from ..config import Config +from ..services.data_collections import DataCollectionService + + +class LabeledButlerFactoryDependency: + """Provides a remote butler factory as a dependency.""" + + def __init__(self) -> None: + self.labeled_butler_factory: LabeledButlerFactory | None = None + + async def initialize( + self, + config: Config, + ) -> None: + """Initialize the dependency.""" + data_repositories = DataCollectionService( + config=config + ).get_data_repositories() + self.labeled_butler_factory = LabeledButlerFactory( + repositories=data_repositories + ) + + async def __call__(self) -> LabeledButlerFactory: + """Return the LabeledButlerFactory instance.""" + if self.labeled_butler_factory is None: + raise RuntimeError( + "LabeledButlerFactoryDependency is not initialized" + ) + return self.labeled_butler_factory + + async def close(self) -> None: + """Close in this case has no effect.""" + + +labeled_butler_factory_dependency = LabeledButlerFactoryDependency() +"""The dependency that will return the LabeledButlerFactoryDependency.""" diff --git a/src/sia/dependencies/query_params.py b/src/sia/dependencies/query_params.py new file mode 100644 index 0000000..39e1b39 --- /dev/null +++ b/src/sia/dependencies/query_params.py @@ -0,0 +1,64 @@ +"""Provides functions to get instances of ParamFactory.""" + +from collections import defaultdict + +from fastapi import Request + +from ..constants import SINGLE_PARAMS +from ..models.sia_query_params import SIAQueryParams + + +async def sia_post_params_dependency( + *, + request: Request, +) -> SIAQueryParams: + """Dependency to parse the POST parameters for the SIA query. + + Parameters + ---------- + request + The request object. + + Returns + ------- + SIAQueryParams + The parameters for the SIA query. + + Raises + ------ + ValueError + If the method is not POST. + """ + if request.method != "POST": + raise ValueError("sia_post_params_dependency used for non-POST route") + content_type = request.headers.get("Content-Type", "") + params_dict: dict[str, list[str]] = defaultdict(list) + + # Handle JSON Content-Type + # This isn't required by the SIA spec, but it may be useful for + # deugging, for future expansion the spec and for demonstration purposes. + if "application/json" in content_type: + json_data = await request.json() + for key, value in json_data.items(): + lower_key = key.lower() + if isinstance(value, list): + params_dict[lower_key].extend(str(v) for v in value) + else: + params_dict[lower_key].append(str(value)) + + else: # Assume form data + form_data = await request.form() + for key, value in form_data.multi_items(): + if not isinstance(value, str): + raise TypeError("File upload not supported") + lower_key = key.lower() + params_dict[lower_key].append(value) + + converted_params_dict = {} + for key, value in params_dict.items(): + if key in SINGLE_PARAMS: + converted_params_dict[key] = value[0] + else: + converted_params_dict[key] = value + + return SIAQueryParams.from_dict(converted_params_dict) diff --git a/src/sia/dependencies/token.py b/src/sia/dependencies/token.py new file mode 100644 index 0000000..e88320b --- /dev/null +++ b/src/sia/dependencies/token.py @@ -0,0 +1,30 @@ +"""FastAPI dependencies for handling tokens.""" + +from fastapi import Header + + +async def optional_auth_delegated_token_dependency( + x_auth_request_token: str | None = Header( + default=None, include_in_schema=False + ), +) -> str | None: + """Make auth_delegated_token_dependency optional. + The use-case for this is for a Direct Butler query where we don't need to + delegate the token. + + Parameters + ---------- + x_auth_request_token + The delegated token. + + Returns + ------- + Optional[str] + The delegated token or None if it is not provided. + """ + if x_auth_request_token is None: + return None + + from safir.dependencies.gafaelfawr import auth_delegated_token_dependency + + return await auth_delegated_token_dependency(x_auth_request_token) diff --git a/src/sia/exceptions.py b/src/sia/exceptions.py new file mode 100644 index 0000000..769f2a0 --- /dev/null +++ b/src/sia/exceptions.py @@ -0,0 +1,270 @@ +"""VOTable exceptions and exception handler to format an error into a valid +VOTAble. +""" + +import functools +import traceback +from collections.abc import Callable +from pathlib import Path +from typing import ParamSpec, TypeVar + +import structlog +from fastapi import FastAPI, Request, Response +from fastapi.exceptions import HTTPException, RequestValidationError +from fastapi.templating import Jinja2Templates + +from .config import config + +_TEMPLATES = Jinja2Templates( + directory=str(Path(__file__).resolve().parent / "templates") +) + +logger = structlog.get_logger(config.name) + +# Module may be slightly too long, in the future we may want to break it up + + +class VOTableError(HTTPException): + """Exception for VOTable errors.""" + + def __init__( + self, detail: str = "Uknown error occured", status_code: int = 400 + ) -> None: + super().__init__(detail=detail, status_code=status_code) + + def __str__(self) -> str: + return f"{self.detail}" + + +class UsageFaultError(VOTableError): + """Exception for invalid input. + + Attributes + ---------- + detail : str + The error message. + status_code : int + The status code for the exception + """ + + def __init__( + self, detail: str = "Invalid input", status_code: int = 400 + ) -> None: + self.detail = f"UsageFault: {detail}" + self.status_code = status_code + super().__init__(detail=self.detail, status_code=self.status_code) + + +class TransientFaultError(VOTableError): + """Exception for service temporarily unavailable. + + Attributes + ---------- + detail : str + The error message. + status_code : int + The status code for the exception + """ + + def __init__( + self, + detail: str = "Service is not currently able to function", + status_code: int = 400, + ) -> None: + self.detail = f"TransientFault: {detail}" + self.status_code = status_code + super().__init__(detail=self.detail, status_code=self.status_code) + + +class FatalFaultError(VOTableError): + """Exception for service cannot perform requested action. + + Attributes + ---------- + detail : str + The error message. + status_code : int + The status code for the exception + """ + + def __init__( + self, + detail: str = "Service cannot perform requested action", + status_code: int = 400, + ) -> None: + self.detail = f"FatalFault: {detail}" + self.status_code = status_code + super().__init__(detail=self.detail, status_code=self.status_code) + + +class DefaultFaultError(VOTableError): + """General exception for errors not covered above. + + Attributes + ---------- + detail : str + The error message. + status_code : int + The status code for the exception + """ + + def __init__( + self, detail: str = "General error", status_code: int = 400 + ) -> None: + self.detail = f"DefaultFault: {detail}" + self.status_code = status_code + super().__init__(detail=self.detail, status_code=self.status_code) + + +async def votable_exception_handler( + request: Request, exc: Exception +) -> Response: + """Handle exceptions that should be returned as VOTable errors. + Produces a VOTable error as a TemplateResponse with the error message. + + Parameters + ---------- + request + The incoming request. + exc + The exception to handle. + + Returns + ------- + Response + The VOTAble error response. + """ + + async def _extract_error_message(excpt: RequestValidationError) -> str: + """Extract the error message from a RequestValidationError. + + Parameters + ---------- + excpt + The RequestValidationError to extract the error message from. + + """ + total_messages = [] + for error in excpt.errors(): + try: + loc = error["loc"][1] + msg = error.get("msg", "Validation error") + input_value = error.get("input", None) + message = f"Validation of '{loc}' failed: {msg}." + if input_value: + message += f" Got: {input_value}." + except IndexError: + message = error["msg"] + total_messages.append(message) + return " ".join(total_messages) + # We are returning the messages separated by a space in the INFO + # element. This is not very readable for a user, but I think the spec + # only allows a single INFO, so maybe there isn't a better way + + logger.error( + "Error during query processing", + error_type=type(exc).__name__, + error_message=str(exc), + path=request.url.path, + method=request.method, + ) + + if isinstance(exc, RequestValidationError): + error_message = await _extract_error_message(exc) + exc = UsageFaultError(detail=error_message) + elif isinstance(exc, ValueError): + error_message = str(exc) + exc = UsageFaultError(detail=error_message) + elif not isinstance( + exc, + UsageFaultError + | TransientFaultError + | FatalFaultError + | DefaultFaultError, + ): + exc = DefaultFaultError(detail=str(exc)) + + response = _TEMPLATES.TemplateResponse( + request, + "votable_error.xml", + { + "request": request, + "error_message": str(exc), + }, + media_type="application/xml", + ) + response.status_code = 400 + return response + + +def configure_exception_handlers(app: FastAPI) -> None: + """Configure the exception handlers for the application. + Handle by formatting as VOTable with the appropriate error message. + + Parameters + ---------- + app + The FastAPI application instance. + """ + + @app.exception_handler(VOTableError) + @app.exception_handler(RequestValidationError) + @app.exception_handler(Exception) + async def custom_exception_handler( + request: Request, exc: Exception + ) -> Response: + """Handle exceptions that should be returned as VOTable errors. + + Parameters + ---------- + request + The incoming request. + exc + The exception to handle. + """ + return await votable_exception_handler(request, exc) + + +R = TypeVar("R") # Return type +P = ParamSpec("P") # Parameters + + +def handle_exceptions(func: Callable[P, R]) -> Callable[P, R]: + """Handle exceptions in the decorated function by logging + and then formatting as a VOTable. + + Parameters + ---------- + func + The function to decorate. + """ + + @functools.wraps(func) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + """Wrap function to handle its exceptions.""" + try: + return func(*args, **kwargs) + except Exception as exc: + stack_trace = "".join( + traceback.format_exception(type(exc), exc, exc.__traceback__) + ) + logger.exception( + "Error during query processing", + error_type=type(exc).__name__, + error_message=str(exc), + stack_trace=stack_trace, + ) + if isinstance( + exc, + UsageFaultError + | TransientFaultError + | FatalFaultError + | DefaultFaultError, + ): + raise exc from exc + + if isinstance(exc, ValueError | RequestValidationError): + raise UsageFaultError(detail=str(exc)) from exc + raise DefaultFaultError(detail=str(exc)) from exc + + return wrapper diff --git a/src/sia/factory.py b/src/sia/factory.py new file mode 100644 index 0000000..f5ff67c --- /dev/null +++ b/src/sia/factory.py @@ -0,0 +1,164 @@ +"""Component factory and process-wide status for mobu.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Self + +from lsst.daf.butler.registry import RegistryDefaults +from structlog.stdlib import BoundLogger + +from .config import Config, config + +__all__ = ["Factory", "ProcessContext"] + +import structlog +from lsst.daf.butler import Butler, LabeledButlerFactory + +from .dependencies.labeled_butler_factory import ( + labeled_butler_factory_dependency, +) +from .exceptions import FatalFaultError +from .models.butler_type import ButlerType +from .models.data_collections import ButlerDataCollection +from .services.data_collections import DataCollectionService + + +@dataclass(frozen=True, slots=True) +class ProcessContext: + """Per-process application context. + + This object caches all of the per-process singletons that can be reused + for every request. + """ + + config: Config + """SIA's configuration.""" + + labeled_butler_factory: LabeledButlerFactory | None + """The Labeled Butler factory.""" + + @classmethod + async def create(cls) -> Self: + labeled_butler_factory = ( + await labeled_butler_factory_dependency() + if config.butler_type is ButlerType.REMOTE + else None + ) + return cls( + config=config, labeled_butler_factory=labeled_butler_factory + ) + + async def aclose(self) -> None: + """Close any resources held by the context.""" + + +class Factory: + """Component factory for sia. + + Uses the contents of a `ProcessContext` to construct the components of an + application on demand. + + Parameters + ---------- + process_context + Shared process context. + """ + + def __init__( + self, + process_context: ProcessContext, + logger: BoundLogger | None = None, + ) -> None: + self._process_context = process_context + self._logger = logger if logger else structlog.get_logger("mobu") + + def create_butler( + self, + config_path: str, + butler_collection: ButlerDataCollection, + token: str | None = None, + ) -> Butler: + """Create a Butler instance. + + Parameters + ---------- + config_path + The path to the Butler configuration. + butler_collection + The Butler data collection. + token + The token to use for the Butler instance. + + Returns + ------- + Butler + The Butler instance. + """ + app_config = self._process_context.config + if not config_path: + raise ValueError( + "No Butler configuration file configured " + ) + + if app_config.butler_type is ButlerType.DIRECT: + if not butler_collection.repository: + raise ValueError( + "No Butler repository configured " + ) + + butler = Butler.from_config( + str(butler_collection.repository), writeable=False + ) + else: + if not butler_collection.label: + raise FatalFaultError( + detail="No Butler label configured