From b2dc04bb1620c098d8ddcc60b8fa36a06d0515fc Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 18 Aug 2023 17:30:11 +0300 Subject: [PATCH 001/110] Version 2.9.0 --- CHANGES.rst | 9 +++++++++ query_exporter/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 76ecbcd..b8f3a95 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,12 @@ +v2.9.0 - 2023-08-18 +=================== + +- Add `increment` flag for counter metrics (#124). +- Rework project setup. +- [docker] Add `pymssql` package (#133). +- [docker] Fix setup for Microsoft repository. + + v2.8.3 - 2022-07-16 =================== diff --git a/query_exporter/__init__.py b/query_exporter/__init__.py index 8fc1899..d9804cf 100644 --- a/query_exporter/__init__.py +++ b/query_exporter/__init__.py @@ -1,3 +1,3 @@ """Export Prometheus metrics generated from SQL queries.""" -__version__ = "2.8.3" +__version__ = "2.9.0" From 4c9f389bf34b75ee10bba7cd9b334d4aad8c3830 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 18 Aug 2023 17:32:03 +0300 Subject: [PATCH 002/110] [snap] fix build --- CHANGES.rst | 2 +- snap/snapcraft.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b8f3a95..587c037 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ v2.9.0 - 2023-08-18 - Add `increment` flag for counter metrics (#124). - Rework project setup. - [docker] Add `pymssql` package (#133). -- [docker] Fix setup for Microsoft repository. +- [docker] Fix setup for Microsoft repository (#159). v2.8.3 - 2022-07-16 diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 425e962..3785ece 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -52,6 +52,7 @@ parts: source: . source-type: git python-packages: + - . - ibm-db-sa; platform_machine == 'x86_64' - ibm-db-sa; platform_machine == 'ppc64le' - ibm-db-sa; platform_machine == 's390x' @@ -62,6 +63,7 @@ parts: - libmysqlclient-dev - libpq-dev - libssl-dev + - pkg-config - unixodbc-dev stage-packages: - libmysqlclient21 From c1764449ed5e290a71b1b21877aac67c81646057 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sun, 27 Aug 2023 17:37:52 +0200 Subject: [PATCH 003/110] [snap] udpate metadata --- snap/snapcraft.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 3785ece..414b030 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -24,9 +24,13 @@ description: | - Microsoft SQL Server (`mssql://`) - IBM DB2 (`db2://`) on supported architectures (x86_64, ppc64le and s390x) - -license: GPL-3.0+ icon: logo.svg +license: GPL-3.0+ +website: https://github.com/albertodonato/query-exporter +source-code: https://github.com/albertodonato/query-exporter +contact: https://github.com/albertodonato/query-expoter/issues +issues: https://github.com/albertodonato/query-exporter/issues + confinement: strict grade: stable base: core22 From 992724d3356b1e8be35fb0034b3572a444ff009c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Fern=C3=A1ndez?= <39913333+darioef@users.noreply.github.com> Date: Mon, 9 Oct 2023 17:14:13 -0300 Subject: [PATCH 004/110] Dockerfile support for arm64 arch (#160) --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9bf621c..8b5e24b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ RUN apt-get install -y --no-install-recommends \ build-essential \ curl \ default-libmysqlclient-dev \ + freetds-dev \ + libkrb5-dev \ libpq-dev \ pkg-config \ unixodbc-dev \ @@ -17,14 +19,14 @@ ENV PATH="/virtualenv/bin:$PATH" RUN pip install \ /srcdir \ cx-Oracle \ - ibm-db-sa \ + "ibm-db-sa; platform_machine == 'x86_64' or platform_machine == 'ppc64le' or platform_machine == 's390x'" \ mysqlclient \ psycopg2 \ pymssql \ pyodbc RUN curl \ - https://download.oracle.com/otn_software/linux/instantclient/instantclient-basiclite-linuxx64.zip \ + https://download.oracle.com/otn_software/linux/instantclient/instantclient-basiclite-linux$(arch | sed -e 's/x86_64/x64/g; s/aarch64/-arm64/g').zip \ -o instantclient.zip RUN unzip instantclient.zip RUN mkdir -p /opt/oracle/instantclient From bcf4aa8c484d4a864bb96edfa97c846749b0269e Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 9 Oct 2023 22:16:57 +0200 Subject: [PATCH 005/110] [github] add python3.12 --- .github/workflows/ci.yml | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63f1d93..f4c84ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,7 @@ jobs: python-version: - "3.10" - "3.11" + - "3.12" steps: - name: Repository checkout uses: actions/checkout@v3 diff --git a/pyproject.toml b/pyproject.toml index ee845a5..790a25a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ classifiers = [ "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Database", "Topic :: System :: Monitoring", "Topic :: Utilities", From e019e9659e7518934728460948ebc5480f7fb96d Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 9 Oct 2023 22:42:51 +0200 Subject: [PATCH 006/110] disable python3.12 for now --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4c84ec..63f1d93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,6 @@ jobs: python-version: - "3.10" - "3.11" - - "3.12" steps: - name: Repository checkout uses: actions/checkout@v3 From 363005d0aae55e2e3b1d00191c2cbd13e4417a49 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 28 Oct 2023 09:40:22 +0200 Subject: [PATCH 007/110] update to prometheus-aioexporter 2.0 (#165) --- pyproject.toml | 2 +- query_exporter/config.py | 46 +++++++++++++++++++++------------------- query_exporter/db.py | 3 ++- query_exporter/loop.py | 7 +++--- tests/config_test.py | 10 ++++----- tox.ini | 3 ++- 6 files changed, 37 insertions(+), 34 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 790a25a..19ddbb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "argcomplete", "croniter", "jsonschema", - "prometheus-aioexporter>=1.5.1", + "prometheus-aioexporter>=2", "prometheus-client", "python-dateutil", "PyYAML", diff --git a/query_exporter/config.py b/query_exporter/config.py index d6f6912..19a9d33 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -2,7 +2,6 @@ from collections import defaultdict from collections.abc import Mapping -from copy import deepcopy from dataclasses import ( dataclass, field, @@ -40,26 +39,26 @@ # metric for counting database errors DB_ERRORS_METRIC_NAME = "database_errors" _DB_ERRORS_METRIC_CONFIG = MetricConfig( - DB_ERRORS_METRIC_NAME, - "Number of database errors", - "counter", - {"labels": []}, + name=DB_ERRORS_METRIC_NAME, + description="Number of database errors", + type="counter", ) + # metric for counting performed queries QUERIES_METRIC_NAME = "queries" _QUERIES_METRIC_CONFIG = MetricConfig( - QUERIES_METRIC_NAME, - "Number of database queries", - "counter", - {"labels": ["query", "status"]}, + name=QUERIES_METRIC_NAME, + description="Number of database queries", + type="counter", + labels=("query", "status"), ) # metric for counting queries execution latency QUERY_LATENCY_METRIC_NAME = "query_latency" _QUERY_LATENCY_METRIC_CONFIG = MetricConfig( - QUERY_LATENCY_METRIC_NAME, - "Query execution latency", - "histogram", - {"labels": ["query"]}, + name=QUERY_LATENCY_METRIC_NAME, + description="Query execution latency", + type="histogram", + labels=("query",), ) GLOBAL_METRICS = frozenset( [DB_ERRORS_METRIC_NAME, QUERIES_METRIC_NAME, QUERY_LATENCY_METRIC_NAME] @@ -167,20 +166,23 @@ def _get_metrics( _QUERIES_METRIC_CONFIG, _QUERY_LATENCY_METRIC_CONFIG, ): - # make a copy since labels are not immutable - metric_config = deepcopy(metric_config) - metric_config.config["labels"].extend(extra_labels) - metric_config.config["labels"].sort() - configs[metric_config.name] = metric_config + configs[metric_config.name] = MetricConfig( + metric_config.name, + metric_config.description, + metric_config.type, + labels=set(metric_config.labels) | extra_labels, + config=metric_config.config, + ) # other metrics for name, config in metrics.items(): _validate_metric_config(name, config, extra_labels) metric_type = config.pop("type") - config.setdefault("labels", []).extend(extra_labels) - config["labels"].sort() + labels = set(config.pop("labels", ())) | extra_labels config["expiration"] = _convert_interval(config.get("expiration")) description = config.pop("description", "") - configs[name] = MetricConfig(name, description, metric_type, config) + configs[name] = MetricConfig( + name, description, metric_type, labels=labels, config=config + ) return configs @@ -255,7 +257,7 @@ def _metric_labels(labels: list[str]) -> list[str]: return sorted(set(labels) - extra_labels) return [ - QueryMetric(name, _metric_labels(metrics[name].config["labels"])) + QueryMetric(name, _metric_labels(metrics[name].labels)) for name in config["metrics"] ] diff --git a/query_exporter/db.py b/query_exporter/db.py index 0825343..bbdb439 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -9,6 +9,7 @@ from typing import ( Any, cast, + Iterable, NamedTuple, ) @@ -128,7 +129,7 @@ class QueryMetric(NamedTuple): """Metric details for a Query.""" name: str - labels: list[str] + labels: Iterable[str] class QueryResults(NamedTuple): diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 8e0e33c..22d8ba2 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -15,7 +15,8 @@ from croniter import croniter from dateutil.tz import gettz from prometheus_aioexporter import MetricsRegistry -from prometheus_client import Metric +from prometheus_client import Counter +from prometheus_client.metrics import MetricWrapperBase from toolrack.aio import ( PeriodicCall, TimedCall, @@ -269,11 +270,11 @@ def _update_metric( self._last_seen.update(name, all_labels, self._timestamp()) def _update_metric_value( - self, metric: Metric, method: str, value: Any + self, metric: MetricWrapperBase, method: str, value: Any ) -> None: if metric._type == "counter" and method == "set": # counters can only be incremented, directly set the underlying value - metric._value.set(value) + cast(Counter, metric)._value.set(value) else: getattr(metric, method)(value) diff --git a/tests/config_test.py b/tests/config_test.py index 539565d..68ec4c3 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -394,23 +394,21 @@ def test_load_metrics_section(self, logger, write_config): metric1 = result.metrics["metric1"] assert metric1.type == "summary" assert metric1.description == "metric one" - assert metric1.config == { - "labels": ["database", "label1", "label2"], - "expiration": 120, - } + assert metric1.labels == ("database", "label1", "label2") + assert metric1.config == {"expiration": 120} metric2 = result.metrics["metric2"] assert metric2.type == "histogram" assert metric2.description == "metric two" + assert metric2.labels == ("database",) assert metric2.config == { - "labels": ["database"], "buckets": [10, 100, 1000], "expiration": None, } metric3 = result.metrics["metric3"] assert metric3.type == "enum" assert metric3.description == "metric three" + assert metric3.labels == ("database",) assert metric3.config == { - "labels": ["database"], "states": ["on", "off"], "expiration": 100, } diff --git a/tox.ini b/tox.ini index 08b8615..42c5da7 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,10 @@ commands = [testenv:check] deps = + .[testing] mypy commands = - mypy -p query_exporter {posargs} + mypy query_exporter {posargs} [testenv:coverage] deps = From e9b289b77a2f5ddd2e4107cb7ee7fef5e6606d51 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 28 Oct 2023 09:47:26 +0200 Subject: [PATCH 008/110] [snap] [docker] add ClickHouse support --- Dockerfile | 1 + snap/snapcraft.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 8b5e24b..4cfc581 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ ENV PATH="/virtualenv/bin:$PATH" RUN pip install \ /srcdir \ cx-Oracle \ + clickhouse-sqlalchemy \ "ibm-db-sa; platform_machine == 'x86_64' or platform_machine == 'ppc64le' or platform_machine == 's390x'" \ mysqlclient \ psycopg2 \ diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 414b030..ec5d2ed 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -24,6 +24,7 @@ description: | - Microsoft SQL Server (`mssql://`) - IBM DB2 (`db2://`) on supported architectures (x86_64, ppc64le and s390x) + - ClickHouse (`clickhouse+native://`) icon: logo.svg license: GPL-3.0+ website: https://github.com/albertodonato/query-exporter @@ -57,6 +58,7 @@ parts: source-type: git python-packages: - . + - clickhouse-sqlalchemy - ibm-db-sa; platform_machine == 'x86_64' - ibm-db-sa; platform_machine == 'ppc64le' - ibm-db-sa; platform_machine == 's390x' From 84ae66cb9cebd3a891f7bf5092ef881135323e59 Mon Sep 17 00:00:00 2001 From: Shubham kashyap <110350667+Shubhamkashyap1601@users.noreply.github.com> Date: Sat, 28 Oct 2023 13:26:04 +0530 Subject: [PATCH 009/110] Update README.rst (#167) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 67ba42e..9f61950 100644 --- a/README.rst +++ b/README.rst @@ -236,7 +236,7 @@ Each query definition can have the following keys: Names must match those defined in the ``databases`` section. Metrics are automatically tagged with the ``database`` label so that - indipendent series are generated for each database that a query is run on. + independent series are generated for each database that a query is run on. ``interval``: the time interval at which the query is run. From 7fb254532067aef8b8c09a57af0dadb58d6ddd33 Mon Sep 17 00:00:00 2001 From: paman6415 <114017588+paman6415@users.noreply.github.com> Date: Sat, 28 Oct 2023 13:26:17 +0530 Subject: [PATCH 010/110] Update README.rst (#166) --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 9f61950..44a5109 100644 --- a/README.rst +++ b/README.rst @@ -94,10 +94,10 @@ A sample configuration file for the application looks like this: ``databases`` section ~~~~~~~~~~~~~~~~~~~~~ -This section contains defintions for databases to connect to. Key names are +This section contains definitions for databases to connect to. Key names are arbitrary and only used to reference databases in the ``queries`` section. -Each database defintions can have the following keys: +Each database definitions can have the following keys: ``dsn``: database connection details. From af92755053677ac88514743553ea0c2c66b80e82 Mon Sep 17 00:00:00 2001 From: daulatojha17 <117494705+daulatojha17@users.noreply.github.com> Date: Sat, 28 Oct 2023 15:10:16 +0530 Subject: [PATCH 011/110] fix typo README.rst (#168) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 44a5109..af9d2a2 100644 --- a/README.rst +++ b/README.rst @@ -547,7 +547,7 @@ The snap has support for connecting the following databases: Run in Docker ------------- -``query-exporter`` can be run inside Docker_ containers, and is availble from +``query-exporter`` can be run inside Docker_ containers, and is available from the `Docker Hub`_:: docker run -p 9560:9560/tcp -v "$CONFIG_FILE:/config.yaml" --rm -it adonato/query-exporter:latest From 058f7309ee3e0e83c403855293cfac27b1d06220 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 28 Oct 2023 16:38:25 +0200 Subject: [PATCH 012/110] Version 2.9.1 --- CHANGES.rst | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 587c037..c12f37a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,9 +1,17 @@ +v2.9.1 - 2023-10-28 +=================== + +- Update dependency to ``prometheus-aioexporter`` 2.0. +- [snap] Add support for ClickHouse. +- [docker] Add support for ClickHouse. + + v2.9.0 - 2023-08-18 =================== -- Add `increment` flag for counter metrics (#124). +- Add ``increment`` flag for counter metrics (#124). - Rework project setup. -- [docker] Add `pymssql` package (#133). +- [docker] Add ``pymssql`` package (#133). - [docker] Fix setup for Microsoft repository (#159). @@ -17,16 +25,16 @@ v2.8.3 - 2022-07-16 v2.8.2 - 2022-07-16 =================== -- Require Python 3.10 -- [snap] Change base to core22 -- [docker] Use Python 3.10 -- [docker] Base on Debian 11 +- Require Python 3.10. +- [snap] Change base to core22. +- [docker] Use Python 3.10. +- [docker] Base on Debian 11. v2.8.1 - 2022-02-18 =================== -- Require `sqlalchemy_aio` 0.17.0, drop workaround for previous versions +- Require ``sqlalchemy_aio`` 0.17.0, drop workaround for previous versions (#105). From ee5318197686b71c3b87eea0be1e22095876799f Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 28 Oct 2023 16:41:52 +0200 Subject: [PATCH 013/110] update version --- query_exporter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query_exporter/__init__.py b/query_exporter/__init__.py index d9804cf..9c29a93 100644 --- a/query_exporter/__init__.py +++ b/query_exporter/__init__.py @@ -1,3 +1,3 @@ """Export Prometheus metrics generated from SQL queries.""" -__version__ = "2.9.0" +__version__ = "2.9.1" From 2467a2947944bf47c4a502886ae7e53be6009534 Mon Sep 17 00:00:00 2001 From: paman6415 <114017588+paman6415@users.noreply.github.com> Date: Sat, 28 Oct 2023 23:24:43 +0530 Subject: [PATCH 014/110] Update CHANGES.rst (#170) --- CHANGES.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c12f37a..e73c1e2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -41,7 +41,7 @@ v2.8.1 - 2022-02-18 v2.8.0 - 2022-01-18 =================== -- Add support for paramters matrix in queries. +- Add support for parameters matrix in queries. - Allow freetext name for databases in config (#99). @@ -106,7 +106,7 @@ v2.4.0 - 2020-06-20 =================== - Add a ``query_latency`` metric to track query execution times. This is - lebeled by database and query name (#46). + labeled by database and query name (#46). v2.3.0 - 2020-06-04 @@ -176,7 +176,7 @@ v2.0.0 - 2020-02-02 - Only named parameters with the ``:param`` style are now supported, queries using positional parameters or other styles of named parameters need to be - udpated. + updated. - Literal ``:`` at the beginning of a word need to be escaped (with backslash) to avoid confusion with parameter markers. Colons that appear inside words don't need to be escaped. @@ -226,13 +226,13 @@ v1.8.0 - 2019-05-25 - Disable queries that will certainly always fail (e.g. because of invalid. returned column names/number) (#6). - Support disconnecting from after each query (#8). -- Rework tests to use actualy SQLite in-memory databases instead of fakes. +- Rework tests to use actually SQLite in-memory databases instead of fakes. v1.7.0 - 2019-04-07 =================== -- Add a ``queries`` and ``database_errors`` metrics lebeled by database (#1). +- Add a ``queries`` and ``database_errors`` metrics labeled by database (#1). - Support database DSNs defined as ``env:`` to supply the dns from the environment (#5). @@ -256,7 +256,7 @@ v1.4.0 - 2018-06-08 =================== - Support for python3.7. -- Use asynctest for asyncronous tests. +- Use asynctest for asynchronous tests. - Updated toolrack dependency. From b599be65682dd9abb967d88c73fe34043924c7a2 Mon Sep 17 00:00:00 2001 From: Mudit Loya <117163483+imperial-chief@users.noreply.github.com> Date: Sat, 28 Oct 2023 23:25:14 +0530 Subject: [PATCH 015/110] Update README.rst (#169) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index af9d2a2..85355f1 100644 --- a/README.rst +++ b/README.rst @@ -310,7 +310,7 @@ Each query definition can have the following keys: - name: Typescript This example will generate 9 queries with all permutations of ``os`` and - ``lang`` paramters. + ``lang`` parameters. ``schedule``: a schedule for executing queries at specific times. From 3947e934cf3861447d2b998fcde6f7dc2ffc0f5d Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 28 Oct 2023 19:57:51 +0200 Subject: [PATCH 016/110] fix main script (fixes #171). --- query_exporter/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query_exporter/main.py b/query_exporter/main.py index 63bd6b2..e8c5aba 100644 --- a/query_exporter/main.py +++ b/query_exporter/main.py @@ -7,10 +7,10 @@ from aiohttp.web import Application from argcomplete import autocomplete from prometheus_aioexporter import ( + InvalidMetricType, MetricConfig, PrometheusExporterScript, ) -from prometheus_aioexporter.metric import InvalidMetricType from toolrack.script import ErrorExitMessage from . import __version__ From 295f0d07efb74bd1f01eafac702f361b479d862c Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 28 Oct 2023 20:00:19 +0200 Subject: [PATCH 017/110] Version 2.9.2 --- CHANGES.rst | 7 +++++++ query_exporter/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e73c1e2..5a3d3ff 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +v2.9.2 - 2023-10-28 +=================== + +- Fix main script (#171). +- Typos fixes in documentation. + + v2.9.1 - 2023-10-28 =================== diff --git a/query_exporter/__init__.py b/query_exporter/__init__.py index 9c29a93..357c4e8 100644 --- a/query_exporter/__init__.py +++ b/query_exporter/__init__.py @@ -1,3 +1,3 @@ """Export Prometheus metrics generated from SQL queries.""" -__version__ = "2.9.1" +__version__ = "2.9.2" From 8d5a16cf7ff8824fd66a149e400232ed92b2276d Mon Sep 17 00:00:00 2001 From: Abhishek Mallick <106394426+Abhishek-Mallick@users.noreply.github.com> Date: Sun, 29 Oct 2023 13:19:19 +0530 Subject: [PATCH 018/110] fix(typs): Fixed minor typos (#172) --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 85355f1..6a945bf 100644 --- a/README.rst +++ b/README.rst @@ -307,7 +307,7 @@ Each query definition can have the following keys: lang: - name: Python3 - name: Java - - name: Typescript + - name: TypeScript This example will generate 9 queries with all permutations of ``os`` and ``lang`` parameters. @@ -490,7 +490,7 @@ The exporter provides a few builtin metrics which can be useful to track query e a histogram with query latencies, per database and query. -In addition, metrics for resources usage for the exporter procecss can be +In addition, metrics for resources usage for the exporter process can be included by passing ``--process-stats`` in the command line. @@ -525,7 +525,7 @@ are supported, via:: sudo snap install query-exporter -The snap provides both the ``query-exporter`` command and a deamon instance of +The snap provides both the ``query-exporter`` command and a daemon instance of the command, managed via a Systemd service. To configure the daemon: From 1744cead06a533ed48b86d6ee58995ff4c9c23f6 Mon Sep 17 00:00:00 2001 From: ilantnt <57303847+ilantnt@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:56:35 +0200 Subject: [PATCH 019/110] Bugfix- Added support for ODBC version 17 using argument (#174) --- Dockerfile | 4 +++- README.rst | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4cfc581..8279cf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,8 @@ RUN mv instantclient*/* /opt/oracle/instantclient FROM python:3.10-slim-bullseye +ARG ODBC_DRIVER_VERSION=18 +ENV ODBC_DRIVER=msodbcsql${ODBC_DRIVER_VERSION} RUN apt-get update && \ apt-get full-upgrade -y && \ @@ -49,7 +51,7 @@ RUN apt-get update && \ curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/microsoft.gpg && \ (. /etc/os-release; echo "deb https://packages.microsoft.com/debian/$VERSION_ID/prod $VERSION_CODENAME main") > /etc/apt/sources.list.d/mssql-release.list && \ apt-get update && \ - ACCEPT_EULA=Y apt-get install -y --no-install-recommends msodbcsql18 && \ + ACCEPT_EULA=Y apt-get install -y --no-install-recommends $ODBC_DRIVER && \ rm -rf /var/lib/apt/lists/* /usr/share/doc /usr/share/man && \ apt-get clean diff --git a/README.rst b/README.rst index 6a945bf..d367043 100644 --- a/README.rst +++ b/README.rst @@ -556,6 +556,9 @@ where ``$CONFIG_FILE`` is the absolute path of the configuration file to use. Note that the image expects the file to be available as ``/config.yaml`` in the container. +For other ODBC versions, build the image with --build-arg VERSION_NUMBER: + docker build --build-arg ODBC_DRIVER_VERSION=17 + The image has support for connecting the following databases: - PostgreSQL (``postgresql://``) From 54cad968ec5af46acd6dccc22be099ff7a90e991 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 1 Nov 2023 17:28:23 +0100 Subject: [PATCH 020/110] update readme --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index d367043..3c4bd88 100644 --- a/README.rst +++ b/README.rst @@ -567,6 +567,7 @@ The image has support for connecting the following databases: - Microsoft SQL Server (``mssql://``) - IBM DB2 (``db2://``) - Oracle (``oracle://``) +- ClickHouse (``clickhouse+native://``) A `Helm chart`_ to run the container in Kubernetes is also available. From cb110cb9e23b47d0a4b8a7a836e10fe2a7e0dddf Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 28 Dec 2023 08:53:19 +0100 Subject: [PATCH 021/110] switch to ruff (#181) --- .github/workflows/ci.yml | 5 +++-- pyproject.toml | 28 +++++++++------------------- query_exporter/config.py | 6 +++--- query_exporter/db.py | 4 ++-- query_exporter/loop.py | 34 ++++++++++++++++++---------------- tests/config_test.py | 8 ++++---- tests/db_test.py | 2 +- tests/loop_test.py | 2 +- tox.ini | 16 +++++----------- 9 files changed, 46 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63f1d93..6ca6977 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Lint run: | - tox -e lint + tox run -e lint check: runs-on: ubuntu-latest @@ -45,7 +45,7 @@ jobs: - name: Check run: | - tox -e check + tox run -e check test: runs-on: ubuntu-latest @@ -54,6 +54,7 @@ jobs: python-version: - "3.10" - "3.11" + - "3.12" steps: - name: Repository checkout uses: actions/checkout@v3 diff --git a/pyproject.toml b/pyproject.toml index 19ddbb0..68a3739 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,10 +16,10 @@ keywords = [ ] license = {file = "LICENSE.txt"} maintainers = [ - {name = "Alberto Donato", email = "alberto.donato@gmail.com"}, + {name = "Alberto Donato", email = "alberto.donato@gmail.com"}, ] authors = [ - {name = "Alberto Donato", email = "alberto.donato@gmail.com"}, + {name = "Alberto Donato", email = "alberto.donato@gmail.com"}, ] requires-python = ">=3.10" classifiers = [ @@ -71,28 +71,18 @@ version = {attr = "query_exporter.__version__"} [tool.setuptools.packages.find] include = ["query_exporter*"] - [tool.setuptools.package-data] -query_exporter = ["schemas/*"] +query_exporter = ["py.typed", "schemas/*"] -[tool.black] +[tool.ruff] line-length = 79 -[tool.isort] -combine_as_imports = true -force_grid_wrap = 2 -force_sort_within_sections = true -from_first = false -include_trailing_comma = true -multi_line_output = 3 -order_by_type = false -profile = "black" -use_parentheses = true +[tool.ruff.lint] +select = ["I", "RUF", "UP"] -[tool.flake8] -ignore = ["E203", "E501", "W503"] -max-line-length = 80 -select = ["C", "E", "F", "W", "B", "B950"] +[tool.ruff.lint.isort] +combine-as-imports = true +force-sort-within-sections = true [tool.pytest.ini_options] asyncio_mode = "auto" diff --git a/query_exporter/config.py b/query_exporter/config.py index 19a9d33..a749cee 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -14,8 +14,8 @@ from pathlib import Path import re from typing import ( - Any, IO, + Any, ) from urllib.parse import ( quote_plus, @@ -27,13 +27,13 @@ import yaml from .db import ( - create_db_engine, DATABASE_LABEL, DataBaseError, InvalidQueryParameters, InvalidQuerySchedule, Query, QueryMetric, + create_db_engine, ) # metric for counting database errors @@ -423,7 +423,7 @@ def _get_parameters_sets(parameters: ParametersConfig) -> list[dict[str, Any]]: def _get_parameters_matrix( - parameters: dict[str, list[dict[str, Any]]] + parameters: dict[str, list[dict[str, Any]]], ) -> list[dict[str, Any]]: """Return parameters combinations from a matrix.""" # first, flatten dict like diff --git a/query_exporter/db.py b/query_exporter/db.py index bbdb439..303a4ab 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -1,6 +1,7 @@ """Database wrapper.""" import asyncio +from collections.abc import Iterable from itertools import chain import logging import sys @@ -8,9 +9,8 @@ from traceback import format_tb from typing import ( Any, - cast, - Iterable, NamedTuple, + cast, ) from croniter import croniter diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 22d8ba2..aa7036e 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -14,7 +14,7 @@ from croniter import croniter from dateutil.tz import gettz -from prometheus_aioexporter import MetricsRegistry +from prometheus_aioexporter import MetricConfig, MetricsRegistry from prometheus_client import Counter from prometheus_client.metrics import MetricWrapperBase from toolrack.aio import ( @@ -23,14 +23,14 @@ ) from .config import ( - Config, DB_ERRORS_METRIC_NAME, QUERIES_METRIC_NAME, QUERY_LATENCY_METRIC_NAME, + Config, ) from .db import ( - DataBase, DATABASE_LABEL, + DataBase, DataBaseConnectError, DataBaseError, Query, @@ -94,14 +94,6 @@ def expire_series( class QueryLoop: """Run database queries and collect metrics.""" - _METRIC_METHODS = { - "counter": "inc", - "gauge": "set", - "histogram": "observe", - "summary": "observe", - "enum": "state", - } - def __init__( self, config: Config, @@ -250,11 +242,6 @@ def _update_metric( elif isinstance(value, Decimal): value = float(value) metric = self._config.metrics[name] - method = self._METRIC_METHODS[metric.type] - if metric.type == "counter" and not metric.config.get( - "increment", True - ): - method = "set" all_labels = {DATABASE_LABEL: database.config.name} all_labels.update(database.config.labels) if labels: @@ -262,6 +249,7 @@ def _update_metric( labels_string = ",".join( f'{label}="{value}"' for label, value in sorted(all_labels.items()) ) + method = self._get_metric_method(metric) self._logger.debug( f'updating metric "{name}" {method} {value} {{{labels_string}}}' ) @@ -269,6 +257,20 @@ def _update_metric( self._update_metric_value(metric, method, value) self._last_seen.update(name, all_labels, self._timestamp()) + def _get_metric_method(self, metric: MetricConfig) -> str: + method = { + "counter": "inc", + "gauge": "set", + "histogram": "observe", + "summary": "observe", + "enum": "state", + }[metric.type] + if metric.type == "counter" and not metric.config.get( + "increment", True + ): + method = "set" + return method + def _update_metric_value( self, metric: MetricWrapperBase, method: str, value: Any ) -> None: diff --git a/tests/config_test.py b/tests/config_test.py index 68ec4c3..8a932f7 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -4,13 +4,13 @@ import yaml from query_exporter.config import ( - _get_parameters_sets, - _resolve_dsn, - ConfigError, DB_ERRORS_METRIC_NAME, GLOBAL_METRICS, - load_config, QUERIES_METRIC_NAME, + ConfigError, + _get_parameters_sets, + _resolve_dsn, + load_config, ) from query_exporter.db import QueryMetric diff --git a/tests/db_test.py b/tests/db_test.py index bcf9a9c..8e67a7c 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -8,7 +8,6 @@ from query_exporter.config import DataBaseConfig from query_exporter.db import ( - create_db_engine, DataBase, DataBaseConnectError, DataBaseError, @@ -22,6 +21,7 @@ QueryMetric, QueryResults, QueryTimeoutExpired, + create_db_engine, ) diff --git a/tests/loop_test.py b/tests/loop_test.py index ba39e65..10396b8 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -178,7 +178,6 @@ async def test_run_query(self, query_tracker, query_loop, registry): async def test_run_scheduled_query( self, mocker, - event_loop, advance_time, query_tracker, registry, @@ -186,6 +185,7 @@ async def test_run_scheduled_query( make_query_loop, ): """Queries are run and update metrics.""" + event_loop = asyncio.get_running_loop() def croniter(*args): while True: diff --git a/tox.ini b/tox.ini index 42c5da7..7521884 100644 --- a/tox.ini +++ b/tox.ini @@ -25,27 +25,21 @@ commands = [testenv:format] deps = - black - isort pyproject-fmt + ruff tox-ini-fmt commands = - isort {[base]lint_files} - black -q {[base]lint_files} + ruff format {[base]lint_files} + ruff check --fix {[base]lint_files} - pyproject-fmt pyproject.toml - tox-ini-fmt tox.ini [testenv:lint] deps = - black - flake8 - flake8-pyproject - isort pyproject-fmt + ruff commands = - isort --check-only --diff {[base]lint_files} - black --check {[base]lint_files} - flake8 {[base]lint_files} + ruff check {[base]lint_files} pyproject-fmt --check pyproject.toml [testenv:run] From 7f1e7aacedf24c77be61f6efa12c0d80d7b02513 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 28 Dec 2023 17:27:41 +0100 Subject: [PATCH 022/110] add metric to track query execution timestamp (fixes #178) (#182) --- README.rst | 7 +++++++ query_exporter/config.py | 16 +++++++++++++++- query_exporter/db.py | 16 +++++++++++++--- query_exporter/loop.py | 16 ++++++++++++++++ tests/db_test.py | 3 +++ tests/loop_test.py | 11 +++++++++-- 6 files changed, 63 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 3c4bd88..3b60e60 100644 --- a/README.rst +++ b/README.rst @@ -433,6 +433,11 @@ For the configuration above, the endpoint would return something like this:: query_latency_created{app="app1",database="db1",query="query1",region="us1"} 1.594544244446163e+09 query_latency_created{app="app1",database="db2",query="query2",region="us2"} 1.5945442444470239e+09 query_latency_created{app="app1",database="db1",query="query2",region="us1"} 1.594544244447551e+09 + # HELP query_timestamp Query last execution timestamp + # TYPE query_timestamp gauge + query_timestamp{app="app1",database="db2",query="query2",region="us2"} 1.594544244446199e+09 + query_timestamp{app="app1",database="db1",query="query1",region="us1"} 1.594544244452181e+09 + query_timestamp{app="app1",database="db1",query="query2",region="us1"} 1.594544244481839e+09 # HELP metric1 A sample gauge # TYPE metric1 gauge metric1{app="app1",database="db1",region="us1"} -3561.0 @@ -489,6 +494,8 @@ The exporter provides a few builtin metrics which can be useful to track query e ``query_latency{database="db",query="q"}``: a histogram with query latencies, per database and query. +``query_timestamp{database="db",query="q"}``: + a gauge with query last execution timestamps, per database and query. In addition, metrics for resources usage for the exporter process can be included by passing ``--process-stats`` in the command line. diff --git a/query_exporter/config.py b/query_exporter/config.py index a749cee..00808fb 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -52,6 +52,14 @@ type="counter", labels=("query", "status"), ) +# metric for tracking last query execution timestamp +QUERY_TIMESTAMP_METRIC_NAME = "query_timestamp" +_QUERY_TIMESTAMP_METRIC_CONFIG = MetricConfig( + name=QUERY_TIMESTAMP_METRIC_NAME, + description="Query last execution timestamp", + type="gauge", + labels=("query",), +) # metric for counting queries execution latency QUERY_LATENCY_METRIC_NAME = "query_latency" _QUERY_LATENCY_METRIC_CONFIG = MetricConfig( @@ -61,7 +69,12 @@ labels=("query",), ) GLOBAL_METRICS = frozenset( - [DB_ERRORS_METRIC_NAME, QUERIES_METRIC_NAME, QUERY_LATENCY_METRIC_NAME] + ( + DB_ERRORS_METRIC_NAME, + QUERIES_METRIC_NAME, + QUERY_LATENCY_METRIC_NAME, + QUERY_TIMESTAMP_METRIC_NAME, + ) ) # regexp for validating environment variables names @@ -165,6 +178,7 @@ def _get_metrics( _DB_ERRORS_METRIC_CONFIG, _QUERIES_METRIC_CONFIG, _QUERY_LATENCY_METRIC_CONFIG, + _QUERY_TIMESTAMP_METRIC_CONFIG, ): configs[metric_config.name] = MetricConfig( metric_config.name, diff --git a/query_exporter/db.py b/query_exporter/db.py index 303a4ab..f73bf87 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -5,7 +5,7 @@ from itertools import chain import logging import sys -from time import perf_counter +from time import perf_counter, time from traceback import format_tb from typing import ( Any, @@ -137,15 +137,20 @@ class QueryResults(NamedTuple): keys: list[str] rows: list[tuple] + timestamp: float | None = None latency: float | None = None @classmethod async def from_results(cls, results: AsyncResultProxy): """Return a QueryResults from results for a query.""" + timestamp = time() conn_info = results._result_proxy.connection.info latency = conn_info.get("query_latency", None) return cls( - await results.keys(), await results.fetchall(), latency=latency + await results.keys(), + await results.fetchall(), + timestamp=timestamp, + latency=latency, ) @@ -161,6 +166,7 @@ class MetricResults(NamedTuple): """Collection of metric results for a query.""" results: list[MetricResult] + timestamp: float | None = None latency: float | None = None @@ -224,7 +230,11 @@ def results(self, query_results: QueryResults) -> MetricResults: ) results.append(metric_result) - return MetricResults(results, latency=query_results.latency) + return MetricResults( + results, + timestamp=query_results.timestamp, + latency=query_results.latency, + ) def _check_schedule(self): if self.interval and self.schedule: diff --git a/query_exporter/loop.py b/query_exporter/loop.py index aa7036e..aa2f3db 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -26,6 +26,7 @@ DB_ERRORS_METRIC_NAME, QUERIES_METRIC_NAME, QUERY_LATENCY_METRIC_NAME, + QUERY_TIMESTAMP_METRIC_NAME, Config, ) from .db import ( @@ -206,6 +207,10 @@ async def _execute_query(self, query: Query, dbname: str): self._update_query_latency_metric( db, query, metric_results.latency ) + if metric_results.timestamp: + self._update_query_timestamp_metric( + db, query, metric_results.timestamp + ) self._increment_queries_count(db, query, "success") async def _remove_if_dooomed(self, query: Query, dbname: str) -> bool: @@ -306,6 +311,17 @@ def _update_query_latency_metric( labels={"query": query.config_name}, ) + def _update_query_timestamp_metric( + self, database: DataBase, query: Query, timestamp: float + ): + """Update timestamp metric for a query on a database.""" + self._update_metric( + database, + QUERY_TIMESTAMP_METRIC_NAME, + timestamp, + labels={"query": query.config_name}, + ) + def _now(self) -> datetime: """Return the current time with local timezone.""" return datetime.now().replace(tzinfo=gettz()) diff --git a/tests/db_test.py b/tests/db_test.py index 8e67a7c..28f7e32 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -1,5 +1,6 @@ import asyncio import logging +import time import pytest from sqlalchemy import create_engine @@ -306,6 +307,7 @@ async def test_from_results(self): assert query_results.keys == ["a", "b"] assert query_results.rows == [(1, 2)] assert query_results.latency is None + assert query_results.timestamp < time.time() @pytest.mark.asyncio async def test_from_results_with_latency(self): @@ -319,6 +321,7 @@ async def test_from_results_with_latency(self): assert query_results.keys == ["a", "b"] assert query_results.rows == [(1, 2)] assert query_results.latency == 1.2 + assert query_results.timestamp < time.time() @pytest.fixture diff --git a/tests/loop_test.py b/tests/loop_test.py index 10396b8..58162eb 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -59,10 +59,11 @@ async def make_query_loop(tmp_path, config_data, registry): def make_loop(): config_file = tmp_path / "config.yaml" config_file.write_text(yaml.dump(config_data), "utf-8") + logger = logging.getLogger() with config_file.open() as fh: - config = load_config(fh, logging.getLogger()) + config = load_config(fh, logger) registry.create_metrics(config.metrics.values()) - query_loop = loop.QueryLoop(config, registry, logging) + query_loop = loop.QueryLoop(config, registry, logger) query_loops.append(query_loop) return query_loop @@ -300,6 +301,9 @@ async def test_run_query_log(self, caplog, query_tracker, query_loop): re_match( r'updating metric "query_latency" observe .* \{database="db",query="q"\}' ), + re_match( + r'updating metric "query_timestamp" set .* \{database="db",query="q"\}' + ), 'updating metric "queries" inc 1 {database="db",query="q",status="success"}', ] @@ -320,6 +324,9 @@ async def test_run_query_log_labels( re_match( r'updating metric "query_latency" observe .* \{database="db",query="q"\}' ), + re_match( + r'updating metric "query_timestamp" set .* \{database="db",query="q"\}' + ), 'updating metric "queries" inc 1 {database="db",query="q",status="success"}', ] From 852f9497c331399e2e1fdfd443057760f0d98b4e Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 17 Jan 2024 11:26:58 +0100 Subject: [PATCH 023/110] [docker] define a volume containing the config file (fixes #158) (#184) --- Dockerfile | 3 ++- README.rst | 16 ++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8279cf5..83c2757 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,5 +63,6 @@ ENV VIRTUAL_ENV="/virtualenv" ENV LD_LIBRARY_PATH="/opt/oracle/instantclient" EXPOSE 9560/tcp +VOLUME /config # IPv6 support is not enabled by default, only bind IPv4 -ENTRYPOINT ["query-exporter", "/config.yaml", "-H", "0.0.0.0"] +ENTRYPOINT ["query-exporter", "/config/config.yaml", "-H", "0.0.0.0"] diff --git a/README.rst b/README.rst index 3b60e60..b382b50 100644 --- a/README.rst +++ b/README.rst @@ -557,14 +557,18 @@ Run in Docker ``query-exporter`` can be run inside Docker_ containers, and is available from the `Docker Hub`_:: - docker run -p 9560:9560/tcp -v "$CONFIG_FILE:/config.yaml" --rm -it adonato/query-exporter:latest + docker run --rm -it -p 9560:9560/tcp -v "$CONFIG_DIR:/config" adonato/query-exporter:latest -where ``$CONFIG_FILE`` is the absolute path of the configuration file to -use. Note that the image expects the file to be available as ``/config.yaml`` -in the container. +where ``$CONFIG_DIR`` is the absolute path of a directory containing a +``config.yaml`` file, the configuration file to use. Alternatively, a volume +name can be specified. + + +A different ODBC driver version to use can be specified during image building, +by passing ``--build-arg ODBC_bVERSION_NUMBER``, e.g.:: + + docker build . --build-arg ODBC_DRIVER_VERSION=17 -For other ODBC versions, build the image with --build-arg VERSION_NUMBER: - docker build --build-arg ODBC_DRIVER_VERSION=17 The image has support for connecting the following databases: From 2347efed5d6236ff322dcdd40fcc9609c827abb4 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 17 Jan 2024 11:38:30 +0100 Subject: [PATCH 024/110] fix tests for changed jsonschema error message --- tests/config_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config_test.py b/tests/config_test.py index 8a932f7..3a5b0b6 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -880,7 +880,7 @@ def test_load_queries_no_metrics(self, logger, config_full, write_config): load_config(fd, logger) assert ( str(err.value) - == "Invalid config at queries/q/metrics: [] is too short" + == "Invalid config at queries/q/metrics: [] should be non-empty" ) def test_load_queries_no_databases( @@ -893,7 +893,7 @@ def test_load_queries_no_databases( load_config(fd, logger) assert ( str(err.value) - == "Invalid config at queries/q/databases: [] is too short" + == "Invalid config at queries/q/databases: [] should be non-empty" ) @pytest.mark.parametrize( From 0f3965727beb8177206696edc1d684819015b3ca Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 23 Jan 2024 00:01:42 +0100 Subject: [PATCH 025/110] drop codecov --- .github/workflows/ci.yml | 9 +-------- README.rst | 5 +---- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ca6977..f2894b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,11 +70,4 @@ jobs: - name: Test run: | - tox -e coverage -- --cov-report xml:coverage.xml - - - name: Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - env_vars: PYTHON - full_report: coverage.xml + tox run -e coverage diff --git a/README.rst b/README.rst index b382b50..bbfa763 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ Export Prometheus metrics from SQL queries ========================================== -|Latest Version| |Build Status| |Coverage Status| |Snap Package| |Docker Pulls| +|Latest Version| |Build Status| |Snap Package| |Docker Pulls| ``query-exporter`` is a Prometheus_ exporter which allows collecting metrics from database queries, at specified time intervals. @@ -603,9 +603,6 @@ A `Helm chart`_ to run the container in Kubernetes is also available. .. |Build Status| image:: https://github.com/albertodonato/query-exporter/workflows/CI/badge.svg :alt: Build Status :target: https://github.com/albertodonato/query-exporter/actions?query=workflow%3ACI -.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/albertodonato/query-exporter/main.svg - :alt: Coverage Status - :target: https://codecov.io/gh/albertodonato/query-exporter .. |Snap Package| image:: https://snapcraft.io/query-exporter/badge.svg :alt: Snap Package :target: https://snapcraft.io/query-exporter From 1bc8aa00f14edd9539751073b13061d9279bce76 Mon Sep 17 00:00:00 2001 From: Andrew Wilkinson Date: Thu, 25 Jan 2024 21:51:02 +0000 Subject: [PATCH 026/110] fix columns names in InvalidResultColumnNames being reported the wrong way round (#185) fix columns names in InvalidResultColumnNames being reported the wrong way round --------- Co-authored-by: Alberto Donato --- query_exporter/db.py | 2 +- tests/db_test.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/query_exporter/db.py b/query_exporter/db.py index f73bf87..4b0ffd5 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -218,7 +218,7 @@ def results(self, query_results: QueryResults) -> MetricResults: if len(expected_keys) != len(result_keys): raise InvalidResultCount(len(expected_keys), len(result_keys)) if result_keys != expected_keys: - raise InvalidResultColumnNames(result_keys, expected_keys) + raise InvalidResultColumnNames(expected_keys, result_keys) results = [] for row in query_results.rows: values = dict(zip(query_results.keys, row)) diff --git a/tests/db_test.py b/tests/db_test.py index 28f7e32..ad9aab0 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -292,8 +292,12 @@ def test_results_wrong_names_with_labels(self): "query", ["db"], [QueryMetric("metric1", ["label1"])], "" ) query_results = QueryResults(["one", "two"], [(1, 2)]) - with pytest.raises(InvalidResultColumnNames): + with pytest.raises(InvalidResultColumnNames) as error: query.results(query_results) + assert str(error.value) == ( + "Wrong column names from query: " + "expected (label1, metric1), got (one, two)" + ) class TestQueryResults: @@ -609,7 +613,7 @@ async def test_execute_query_invalid_names_with_labels(self, db): await db.execute(query) assert ( str(error.value) - == "Wrong column names from query: expected (foo, label), got (label, metric)" + == "Wrong column names from query: expected (label, metric), got (foo, label)" ) assert error.value.fatal @@ -618,7 +622,7 @@ async def test_execute_query_traceback_debug(self, caplog, mocker, db): """Traceback are logged as debug messages.""" query = Query( "query", - 20, + ["db"], [QueryMetric("metric", [])], "SELECT 1 AS metric", ) From f9c37346e5db3c45e4a2d006ff386aa095b400b6 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sun, 28 Jan 2024 18:28:17 +0100 Subject: [PATCH 027/110] Version 2.10.0 --- CHANGES.rst | 11 +++++++++++ pyproject.toml | 2 +- query_exporter/__init__.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5a3d3ff..eda09a8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,14 @@ +v2.10.0 - 2024-01-28 +==================== + +- Fix columns names in InvalidResultColumnNames being reported the wrong way + round (#185). +- Add a metric to track query execution timestamp (#178). +- [docker] Define a volume containing the config file (#158). +- [docker] Add support for ODBC version 17, support alternative versions. +- Switch to ruff for formatting. + + v2.9.2 - 2023-10-28 =================== diff --git a/pyproject.toml b/pyproject.toml index 68a3739..8e72b78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ dependencies = [ ] [project.optional-dependencies] testing = [ - "pytest", + "pytest<8", # XXX https://github.com/pytest-dev/pytest-asyncio/issues/737 "pytest-asyncio", "pytest-mock", ] diff --git a/query_exporter/__init__.py b/query_exporter/__init__.py index 357c4e8..3f0d965 100644 --- a/query_exporter/__init__.py +++ b/query_exporter/__init__.py @@ -1,3 +1,3 @@ """Export Prometheus metrics generated from SQL queries.""" -__version__ = "2.9.2" +__version__ = "2.10.0" From 6661e01be16ed21730d82d321e4a9057ce7da346 Mon Sep 17 00:00:00 2001 From: Florian Mueller <46089838+flo-02-mu@users.noreply.github.com> Date: Sun, 20 Oct 2024 16:14:38 +0200 Subject: [PATCH 028/110] Update the base image to 3.13-slim-bookworm (#197) --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 83c2757..80d26b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-bullseye AS build-image +FROM python:3.13-slim-bookworm AS build-image RUN apt-get update RUN apt-get full-upgrade -y @@ -34,7 +34,7 @@ RUN mkdir -p /opt/oracle/instantclient RUN mv instantclient*/* /opt/oracle/instantclient -FROM python:3.10-slim-bullseye +FROM python:3.13-slim-bookworm ARG ODBC_DRIVER_VERSION=18 ENV ODBC_DRIVER=msodbcsql${ODBC_DRIVER_VERSION} From cd96f391bf7ef8e083b5fcfcc3316451dc35fda7 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sun, 20 Oct 2024 16:15:02 +0200 Subject: [PATCH 029/110] switch back to isort+black+flake8 --- pyproject.toml | 57 ++++++++++++++++++++++------------------ query_exporter/config.py | 4 +-- query_exporter/db.py | 7 +++-- query_exporter/loop.py | 9 ++++--- tests/config_test.py | 8 +++--- tests/db_test.py | 37 +++++++++++++++----------- tox.ini | 16 +++++++---- 7 files changed, 81 insertions(+), 57 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8e72b78..7275b78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,12 @@ keywords = [ "prometheus", "sql", ] -license = {file = "LICENSE.txt"} +license = { file = "LICENSE.txt" } maintainers = [ - {name = "Alberto Donato", email = "alberto.donato@gmail.com"}, + { name = "Alberto Donato", email = "alberto.donato@gmail.com" }, ] authors = [ - {name = "Alberto Donato", email = "alberto.donato@gmail.com"}, + { name = "Alberto Donato", email = "alberto.donato@gmail.com" }, ] requires-python = ">=3.10" classifiers = [ @@ -32,6 +32,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Database", "Topic :: System :: Monitoring", "Topic :: Utilities", @@ -47,42 +48,48 @@ dependencies = [ "prometheus-aioexporter>=2", "prometheus-client", "python-dateutil", - "PyYAML", - "SQLAlchemy<1.4", - "sqlalchemy_aio>=0.17", + "pyyaml", + "sqlalchemy<1.4", + "sqlalchemy-aio>=0.17", "toolrack>=4", ] -[project.optional-dependencies] -testing = [ - "pytest<8", # XXX https://github.com/pytest-dev/pytest-asyncio/issues/737 +optional-dependencies.testing = [ + "pytest", "pytest-asyncio", "pytest-mock", ] -[project.urls] -changelog = "https://github.com/albertodonato/query-exporter/blob/main/CHANGES.rst" -homepage = "https://github.com/albertodonato/query-exporter" -repository = "https://github.com/albertodonato/query-exporter" -[project.scripts] -query-exporter = "query_exporter.main:script" +urls.changelog = "https://github.com/albertodonato/query-exporter/blob/main/CHANGES.rst" +urls.homepage = "https://github.com/albertodonato/query-exporter" +urls.repository = "https://github.com/albertodonato/query-exporter" +scripts.query-exporter = "query_exporter.main:script" [tool.setuptools.dynamic] -version = {attr = "query_exporter.__version__"} +version = { attr = "query_exporter.__version__" } [tool.setuptools.packages.find] -include = ["query_exporter*"] +include = [ "query_exporter*" ] [tool.setuptools.package-data] -query_exporter = ["py.typed", "schemas/*"] +query_exporter = [ "py.typed", "schemas/*" ] -[tool.ruff] +[tool.black] line-length = 79 -[tool.ruff.lint] -select = ["I", "RUF", "UP"] +[tool.isort] +combine_as_imports = true +force_grid_wrap = 2 +force_sort_within_sections = true +from_first = false +include_trailing_comma = true +multi_line_output = 3 +order_by_type = false +profile = "black" +use_parentheses = true -[tool.ruff.lint.isort] -combine-as-imports = true -force-sort-within-sections = true +[tool.flake8] +ignore = [ "E203", "E501", "W503" ] +max-line-length = 80 +select = [ "C", "E", "F", "W", "B", "B950" ] [tool.pytest.ini_options] asyncio_mode = "auto" @@ -93,7 +100,7 @@ show_missing = true skip_covered = true [tool.coverage.run] -source = ["query_exporter"] +source = [ "query_exporter" ] omit = [ "query_exporter/main.py", ] diff --git a/query_exporter/config.py b/query_exporter/config.py index 00808fb..8c39527 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -14,8 +14,8 @@ from pathlib import Path import re from typing import ( - IO, Any, + IO, ) from urllib.parse import ( quote_plus, @@ -27,13 +27,13 @@ import yaml from .db import ( + create_db_engine, DATABASE_LABEL, DataBaseError, InvalidQueryParameters, InvalidQuerySchedule, Query, QueryMetric, - create_db_engine, ) # metric for counting database errors diff --git a/query_exporter/db.py b/query_exporter/db.py index 4b0ffd5..4d32987 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -5,12 +5,15 @@ from itertools import chain import logging import sys -from time import perf_counter, time +from time import ( + perf_counter, + time, +) from traceback import format_tb from typing import ( Any, - NamedTuple, cast, + NamedTuple, ) from croniter import croniter diff --git a/query_exporter/loop.py b/query_exporter/loop.py index aa2f3db..3532b89 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -14,7 +14,10 @@ from croniter import croniter from dateutil.tz import gettz -from prometheus_aioexporter import MetricConfig, MetricsRegistry +from prometheus_aioexporter import ( + MetricConfig, + MetricsRegistry, +) from prometheus_client import Counter from prometheus_client.metrics import MetricWrapperBase from toolrack.aio import ( @@ -23,15 +26,15 @@ ) from .config import ( + Config, DB_ERRORS_METRIC_NAME, QUERIES_METRIC_NAME, QUERY_LATENCY_METRIC_NAME, QUERY_TIMESTAMP_METRIC_NAME, - Config, ) from .db import ( - DATABASE_LABEL, DataBase, + DATABASE_LABEL, DataBaseConnectError, DataBaseError, Query, diff --git a/tests/config_test.py b/tests/config_test.py index 3a5b0b6..7d2891c 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -4,13 +4,13 @@ import yaml from query_exporter.config import ( - DB_ERRORS_METRIC_NAME, - GLOBAL_METRICS, - QUERIES_METRIC_NAME, - ConfigError, _get_parameters_sets, _resolve_dsn, + ConfigError, + DB_ERRORS_METRIC_NAME, + GLOBAL_METRICS, load_config, + QUERIES_METRIC_NAME, ) from query_exporter.db import QueryMetric diff --git a/tests/db_test.py b/tests/db_test.py index ad9aab0..8b9d8a5 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -9,6 +9,7 @@ from query_exporter.config import DataBaseConfig from query_exporter.db import ( + create_db_engine, DataBase, DataBaseConnectError, DataBaseError, @@ -22,7 +23,6 @@ QueryMetric, QueryResults, QueryTimeoutExpired, - create_db_engine, ) @@ -44,9 +44,10 @@ def test_instantiate_missing_engine_module(self, caplog): @pytest.mark.parametrize("dsn", ["foo-bar", "unknown:///db"]) def test_instantiate_invalid_dsn(self, caplog, dsn): """An error is raised if a the provided DSN is invalid.""" - with caplog.at_level(logging.ERROR), pytest.raises( - DataBaseError - ) as error: + with ( + caplog.at_level(logging.ERROR), + pytest.raises(DataBaseError) as error, + ): create_db_engine(dsn) assert str(error.value) == f'Invalid database DSN: "{dsn}"' @@ -412,9 +413,10 @@ async def test_connect_sql_fail(self, caplog): connect_sql=["WRONG"], ) db = DataBase(config) - with caplog.at_level(logging.DEBUG), pytest.raises( - DataBaseQueryError - ) as error: + with ( + caplog.at_level(logging.DEBUG), + pytest.raises(DataBaseQueryError) as error, + ): await db.connect() assert not db.connected assert 'failed executing query "WRONG"' in str(error.value) @@ -567,9 +569,10 @@ async def test_execute_query_invalid_count(self, caplog, db): "SELECT 1 AS metric, 2 AS other", ) await db.connect() - with caplog.at_level(logging.ERROR), pytest.raises( - DataBaseQueryError - ) as error: + with ( + caplog.at_level(logging.ERROR), + pytest.raises(DataBaseQueryError) as error, + ): await db.execute(query) assert ( str(error.value) @@ -630,9 +633,10 @@ async def test_execute_query_traceback_debug(self, caplog, mocker, db): "boom!" ) await db.connect() - with caplog.at_level(logging.DEBUG), pytest.raises( - DataBaseQueryError - ) as error: + with ( + caplog.at_level(logging.DEBUG), + pytest.raises(DataBaseQueryError) as error, + ): await db.execute(query) assert str(error.value) == "boom!" assert not error.value.fatal @@ -659,9 +663,10 @@ async def execute(sql, parameters): db._conn.execute = execute - with caplog.at_level(logging.WARNING), pytest.raises( - QueryTimeoutExpired - ) as error: + with ( + caplog.at_level(logging.WARNING), + pytest.raises(QueryTimeoutExpired) as error, + ): await db.execute(query) assert ( str(error.value) diff --git a/tox.ini b/tox.ini index 7521884..42c5da7 100644 --- a/tox.ini +++ b/tox.ini @@ -25,21 +25,27 @@ commands = [testenv:format] deps = + black + isort pyproject-fmt - ruff tox-ini-fmt commands = - ruff format {[base]lint_files} - ruff check --fix {[base]lint_files} + isort {[base]lint_files} + black -q {[base]lint_files} - pyproject-fmt pyproject.toml - tox-ini-fmt tox.ini [testenv:lint] deps = + black + flake8 + flake8-pyproject + isort pyproject-fmt - ruff commands = - ruff check {[base]lint_files} + isort --check-only --diff {[base]lint_files} + black --check {[base]lint_files} + flake8 {[base]lint_files} pyproject-fmt --check pyproject.toml [testenv:run] From e4cb28a9ecafea6816effe1e18bfe7437417a2c3 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sun, 20 Oct 2024 20:34:33 +0200 Subject: [PATCH 030/110] gh: add python 3.13 to matrix --- .github/workflows/ci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2894b4..688b58f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.13" - name: Install dependencies run: | @@ -30,6 +30,13 @@ jobs: check: runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - "3.10" + - "3.11" + - "3.12" + - "3.13" steps: - name: Repository checkout uses: actions/checkout@v3 @@ -37,7 +44,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | @@ -55,6 +62,7 @@ jobs: - "3.10" - "3.11" - "3.12" + - "3.13" steps: - name: Repository checkout uses: actions/checkout@v3 From ee638b3a38f93cfc674c32ebdbb451cc213f5a26 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 17 Apr 2023 19:52:43 +0200 Subject: [PATCH 031/110] enable strict mode for mypy --- pyproject.toml | 3 +- query_exporter/config.py | 36 ++++----------------- query_exporter/db.py | 70 +++++++++++++++++++++++++++------------- query_exporter/loop.py | 29 +++++++++-------- query_exporter/main.py | 16 +++++---- 5 files changed, 79 insertions(+), 75 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7275b78..08cb3aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,5 +109,4 @@ omit = [ ignore_missing_imports = true install_types = true non_interactive = true -warn_return_any = true -warn_unused_configs = true +strict = true diff --git a/query_exporter/config.py b/query_exporter/config.py index 8c39527..abf000f 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -2,10 +2,7 @@ from collections import defaultdict from collections.abc import Mapping -from dataclasses import ( - dataclass, - field, -) +from dataclasses import dataclass from functools import reduce from importlib import resources import itertools @@ -27,9 +24,8 @@ import yaml from .db import ( - create_db_engine, DATABASE_LABEL, - DataBaseError, + DataBaseConfig, InvalidQueryParameters, InvalidQuerySchedule, Query, @@ -85,24 +81,6 @@ class ConfigError(Exception): """Configuration is invalid.""" -@dataclass(frozen=True) -class DataBaseConfig: - """Configuration for a database.""" - - name: str - dsn: str - connect_sql: list[str] = field(default_factory=list) - labels: dict[str, str] = field(default_factory=dict) - keep_connected: bool = True - autocommit: bool = True - - def __post_init__(self): - try: - create_db_engine(self.dsn) - except DataBaseError as e: - raise ConfigError(str(e)) - - @dataclass(frozen=True) class Config: """Top-level configuration.""" @@ -120,7 +98,7 @@ class Config: def load_config( - config_fd: IO, logger: Logger, env: Environ = os.environ + config_fd: IO[str], logger: Logger, env: Environ = os.environ ) -> Config: """Load YAML config from file.""" data = defaultdict(dict, yaml.safe_load(config_fd)) @@ -202,7 +180,7 @@ def _get_metrics( def _validate_metric_config( name: str, config: dict[str, Any], extra_labels: frozenset[str] -): +) -> None: """Validate a metric configuration stanza.""" if name in GLOBAL_METRICS: raise ConfigError(f'Label name "{name} is reserved for builtin metric') @@ -281,7 +259,7 @@ def _validate_query_config( config: dict[str, Any], database_names: frozenset[str], metric_names: frozenset[str], -): +) -> None: """Validate a query configuration stanza.""" unknown_databases = set(config["databases"]) - database_names if unknown_databases: @@ -397,7 +375,7 @@ def _build_dsn(details: dict[str, Any]) -> str: return url -def _validate_config(config: dict[str, Any]): +def _validate_config(config: dict[str, Any]) -> None: schema_file = resources.files("query_exporter") / "schemas" / "config.yaml" schema = yaml.safe_load(schema_file.read_bytes()) try: @@ -407,7 +385,7 @@ def _validate_config(config: dict[str, Any]): raise ConfigError(f"Invalid config at {path}: {e.message}") -def _warn_if_unused(config: Config, logger: Logger): +def _warn_if_unused(config: Config, logger: Logger) -> None: """Warn if there are unused databases or metrics defined.""" used_dbs: set[str] = set() used_metrics: set[str] = set() diff --git a/query_exporter/db.py b/query_exporter/db.py index 4d32987..fa7e07a 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -2,6 +2,10 @@ import asyncio from collections.abc import Iterable +from dataclasses import ( + dataclass, + field, +) from itertools import chain import logging import sys @@ -10,6 +14,7 @@ time, ) from traceback import format_tb +from types import TracebackType from typing import ( Any, cast, @@ -47,7 +52,7 @@ class DataBaseError(Exception): if `fatal` is True, it means the Query will never succeed. """ - def __init__(self, message: str, fatal: bool = False): + def __init__(self, message: str, fatal: bool = False) -> None: super().__init__(message) self.fatal = fatal @@ -63,7 +68,7 @@ class DataBaseQueryError(DataBaseError): class QueryTimeoutExpired(Exception): """Query execution timeout expired.""" - def __init__(self, query_name: str, timeout: QueryTimeout): + def __init__(self, query_name: str, timeout: QueryTimeout) -> None: super().__init__( f'Execution for query "{query_name}" expired after {timeout} seconds' ) @@ -72,7 +77,7 @@ def __init__(self, query_name: str, timeout: QueryTimeout): class InvalidResultCount(Exception): """Number of results from a query don't match metrics count.""" - def __init__(self, expected: int, got: int): + def __init__(self, expected: int, got: int) -> None: super().__init__( f"Wrong result count from query: expected {expected}, got {got}" ) @@ -95,7 +100,7 @@ def _names(self, names: list[str]) -> str: class InvalidQueryParameters(Exception): """Query parameter names don't match those in query SQL.""" - def __init__(self, query_name: str): + def __init__(self, query_name: str) -> None: super().__init__( f'Parameters for query "{query_name}" don\'t match those from SQL' ) @@ -104,7 +109,7 @@ def __init__(self, query_name: str): class InvalidQuerySchedule(Exception): """Query schedule is wrong or both schedule and interval specified.""" - def __init__(self, query_name: str, message: str): + def __init__(self, query_name: str, message: str) -> None: super().__init__( f'Invalid schedule for query "{query_name}": {message}' ) @@ -118,7 +123,23 @@ def __init__(self, query_name: str, message: str): FATAL_ERRORS = (InvalidResultCount, InvalidResultColumnNames) -def create_db_engine(dsn: str, **kwargs) -> AsyncioEngine: +@dataclass(frozen=True) +class DataBaseConfig: + """Configuration for a database.""" + + name: str + dsn: str + connect_sql: list[str] = field(default_factory=list) + labels: dict[str, str] = field(default_factory=dict) + keep_connected: bool = True + autocommit: bool = True + + def __post_init__(self) -> None: + # raise DatabaseError error if the DSN in invalid + create_db_engine(self.dsn) + + +def create_db_engine(dsn: str, **kwargs: Any) -> AsyncioEngine: """Create the database engine, validating the DSN""" try: return create_engine(dsn, **kwargs) @@ -139,12 +160,12 @@ class QueryResults(NamedTuple): """Results of a database query.""" keys: list[str] - rows: list[tuple] + rows: list[tuple[Any]] timestamp: float | None = None latency: float | None = None @classmethod - async def from_results(cls, results: AsyncResultProxy): + async def from_results(cls, results: AsyncResultProxy) -> "QueryResults": """Return a QueryResults from results for a query.""" timestamp = time() conn_info = results._result_proxy.connection.info @@ -187,7 +208,7 @@ def __init__( interval: int | None = None, schedule: str | None = None, config_name: str | None = None, - ): + ) -> None: self.name = name self.databases = databases self.metrics = metrics @@ -239,7 +260,7 @@ def results(self, query_results: QueryResults) -> MetricResults: latency=query_results.latency, ) - def _check_schedule(self): + def _check_schedule(self) -> None: if self.interval and self.schedule: raise InvalidQuerySchedule( self.name, "both interval and schedule specified" @@ -247,7 +268,7 @@ def _check_schedule(self): if self.schedule and not croniter.is_valid(self.schedule): raise InvalidQuerySchedule(self.name, "invalid schedule format") - def _check_query_parameters(self): + def _check_query_parameters(self) -> None: expr = text(self.sql) query_params = set(expr.compile().params) if set(self.parameters) != query_params: @@ -263,9 +284,9 @@ class DataBase: def __init__( self, - config, + config: DataBaseConfig, logger: logging.Logger = logging.getLogger(), - ): + ) -> None: self.config = config self.logger = logger self._connect_lock = asyncio.Lock() @@ -277,11 +298,13 @@ def __init__( self._setup_query_latency_tracking() - async def __aenter__(self): + async def __aenter__(self) -> "DataBase": await self.connect() return self - async def __aexit__(self, exc_type, exc_value, traceback): + async def __aexit__( + self, exc_type: type, exc_value: Exception, traceback: TracebackType + ) -> None: await self.close() @property @@ -289,7 +312,7 @@ def connected(self) -> bool: """Whether the database is connected.""" return self._conn is not None - async def connect(self): + async def connect(self) -> None: """Connect to the database.""" async with self._connect_lock: if self.connected: @@ -311,7 +334,7 @@ async def connect(self): exc_class=DataBaseQueryError, ) - async def close(self): + async def close(self) -> None: """Close the database connection.""" async with self._connect_lock: if not self.connected: @@ -364,27 +387,28 @@ async def _execute_query(self, query: Query) -> AsyncResultProxy: query.sql, parameters=query.parameters, timeout=query.timeout ) - async def _close(self): + async def _close(self) -> None: # ensure the connection with the DB is actually closed + self._conn: AsyncConnection self._conn.sync_connection.detach() await self._conn.close() self._conn = None self._pending_queries = 0 self.logger.debug(f'disconnected from database "{self.config.name}"') - def _setup_query_latency_tracking(self): + def _setup_query_latency_tracking(self) -> None: engine = self._engine.sync_engine - @event.listens_for(engine, "before_cursor_execute") + @event.listens_for(engine, "before_cursor_execute") # type: ignore def before_cursor_execute( conn, cursor, statement, parameters, context, executemany - ): + ) -> None: conn.info["query_start_time"] = perf_counter() - @event.listens_for(engine, "after_cursor_execute") + @event.listens_for(engine, "after_cursor_execute") # type: ignore def after_cursor_execute( conn, cursor, statement, parameters, context, executemany - ): + ) -> None: conn.info["query_latency"] = perf_counter() - conn.info.pop( "query_start_time" ) diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 3532b89..a4cc014 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -10,6 +10,7 @@ from typing import ( Any, cast, + Iterator, ) from croniter import croniter @@ -60,7 +61,7 @@ def update( name: str, labels: dict[str, str], timestamp: float, - ): + ) -> None: """Update last seen for a metric with a set of labels to given timestamp.""" if not self._expirations.get(name): return @@ -131,18 +132,18 @@ def __init__( else: self._aperiodic_queries.append(query) - async def start(self): + async def start(self) -> None: """Start timed queries execution.""" for query in self._timed_queries: if query.interval: call = PeriodicCall(self._run_query, query) call.start(query.interval) - else: + elif query.schedule is not None: call = TimedCall(self._run_query, query) call.start(self._loop_times_iter(query.schedule)) self._timed_calls[query.name] = call - async def stop(self): + async def stop(self) -> None: """Stop timed query execution.""" coros = (call.stop() for call in self._timed_calls.values()) await asyncio.gather(*coros, return_exceptions=True) @@ -150,7 +151,7 @@ async def stop(self): coros = (db.close() for db in self._databases.values()) await asyncio.gather(*coros, return_exceptions=True) - def clear_expired_series(self): + def clear_expired_series(self) -> None: """Clear metric series that have expired.""" expired_series = self._last_seen.expire_series(self._timestamp()) for name, label_values in expired_series.items(): @@ -158,7 +159,7 @@ def clear_expired_series(self): for values in label_values: metric.remove(*values) - async def run_aperiodic_queries(self): + async def run_aperiodic_queries(self) -> None: """Run queries on request.""" coros = ( self._execute_query(query, dbname) @@ -167,7 +168,7 @@ async def run_aperiodic_queries(self): ) await asyncio.gather(*coros, return_exceptions=True) - def _loop_times_iter(self, schedule: str): + def _loop_times_iter(self, schedule: str) -> Iterator[float | int]: """Wrap a croniter iterator to sync time with the loop clock.""" cron_iter = croniter(schedule, self._now()) while True: @@ -176,12 +177,12 @@ def _loop_times_iter(self, schedule: str): delta = cc - t yield self._loop.time() + delta - def _run_query(self, query: Query): + def _run_query(self, query: Query) -> None: """Periodic task to run a query.""" for dbname in query.databases: self._loop.create_task(self._execute_query(query, dbname)) - async def _execute_query(self, query: Query, dbname: str): + async def _execute_query(self, query: Query, dbname: str) -> None: """'Execute a Query on a DataBase.""" if await self._remove_if_dooomed(query, dbname): return @@ -242,7 +243,7 @@ def _update_metric( name: str, value: Any, labels: Mapping[str, str] | None = None, - ): + ) -> None: """Update value for a metric.""" if value is None: # don't fail is queries that count return NULL @@ -290,7 +291,7 @@ def _update_metric_value( def _increment_queries_count( self, database: DataBase, query: Query, status: str - ): + ) -> None: """Increment count of queries in a status for a database.""" self._update_metric( database, @@ -299,13 +300,13 @@ def _increment_queries_count( labels={"query": query.config_name, "status": status}, ) - def _increment_db_error_count(self, database: DataBase): + def _increment_db_error_count(self, database: DataBase) -> None: """Increment number of errors for a database.""" self._update_metric(database, DB_ERRORS_METRIC_NAME, 1) def _update_query_latency_metric( self, database: DataBase, query: Query, latency: float - ): + ) -> None: """Update latency metric for a query on a database.""" self._update_metric( database, @@ -316,7 +317,7 @@ def _update_query_latency_metric( def _update_query_timestamp_metric( self, database: DataBase, query: Query, timestamp: float - ): + ) -> None: """Update timestamp metric for a query on a database.""" self._update_metric( database, diff --git a/query_exporter/main.py b/query_exporter/main.py index e8c5aba..a71acd4 100644 --- a/query_exporter/main.py +++ b/query_exporter/main.py @@ -22,14 +22,16 @@ from .loop import QueryLoop -class QueryExporterScript(PrometheusExporterScript): +class QueryExporterScript(PrometheusExporterScript): # type: ignore """Periodically run database queries and export results to Prometheus.""" name = "query-exporter" description = __doc__ default_port = 9560 - def configure_argument_parser(self, parser: argparse.ArgumentParser): + def configure_argument_parser( + self, parser: argparse.ArgumentParser + ) -> None: parser.add_argument( "config", type=argparse.FileType("r"), help="configuration file" ) @@ -46,13 +48,13 @@ def configure_argument_parser(self, parser: argparse.ArgumentParser): ) autocomplete(parser) - def configure(self, args: argparse.Namespace): + def configure(self, args: argparse.Namespace) -> None: self.config = self._load_config(args.config) if args.check_only: self.exit() self.create_metrics(self.config.metrics.values()) - async def on_application_startup(self, application: Application): + async def on_application_startup(self, application: Application) -> None: self.logger.info(f"version {__version__} starting up") query_loop = QueryLoop(self.config, self.registry, self.logger) application["exporter"].set_metric_update_handler( @@ -61,17 +63,17 @@ async def on_application_startup(self, application: Application): application["query-loop"] = query_loop await query_loop.start() - async def on_application_shutdown(self, application: Application): + async def on_application_shutdown(self, application: Application) -> None: await application["query-loop"].stop() async def _update_handler( self, query_loop: QueryLoop, metrics: list[MetricConfig] - ): + ) -> None: """Run queries with no specified schedule on each request.""" await query_loop.run_aperiodic_queries() query_loop.clear_expired_series() - def _load_config(self, config_file: IO) -> Config: + def _load_config(self, config_file: IO[str]) -> Config: """Load the application configuration.""" try: config = load_config(config_file, self.logger) From 55f20c80a1c4bd4f18b91c15787b147a6784c23f Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 24 Oct 2024 13:35:56 +0200 Subject: [PATCH 032/110] set scope for asyncio fixture in tests --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 08cb3aa..bee545c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ select = [ "C", "E", "F", "W", "B", "B950" ] [tool.pytest.ini_options] asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" [tool.coverage.report] fail_under = 100.0 From 8f1f568857fdaf459287dc1c3f2d2bc082150a28 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 29 Oct 2024 13:25:11 +0100 Subject: [PATCH 033/110] typing fix --- query_exporter/loop.py | 1 + 1 file changed, 1 insertion(+) diff --git a/query_exporter/loop.py b/query_exporter/loop.py index a4cc014..e355a2c 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -135,6 +135,7 @@ def __init__( async def start(self) -> None: """Start timed queries execution.""" for query in self._timed_queries: + call: TimedCall if query.interval: call = PeriodicCall(self._run_query, query) call.start(query.interval) From 5c2151756f23dd87d2deb75f4cd1c5cd7cdab5a0 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 29 Oct 2024 13:28:10 +0100 Subject: [PATCH 034/110] revert switch back from ruff --- pyproject.toml | 20 ++++---------------- query_exporter/config.py | 2 +- query_exporter/db.py | 2 +- query_exporter/loop.py | 7 +++---- tests/config_test.py | 8 ++++---- tests/db_test.py | 2 +- tox.ini | 16 +++++----------- 7 files changed, 19 insertions(+), 38 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bee545c..84b903a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,24 +72,12 @@ include = [ "query_exporter*" ] [tool.setuptools.package-data] query_exporter = [ "py.typed", "schemas/*" ] -[tool.black] +[tool.ruff] line-length = 79 -[tool.isort] -combine_as_imports = true -force_grid_wrap = 2 -force_sort_within_sections = true -from_first = false -include_trailing_comma = true -multi_line_output = 3 -order_by_type = false -profile = "black" -use_parentheses = true - -[tool.flake8] -ignore = [ "E203", "E501", "W503" ] -max-line-length = 80 -select = [ "C", "E", "F", "W", "B", "B950" ] +lint.select = [ "I", "RUF", "UP" ] +lint.isort.combine-as-imports = true +lint.isort.force-sort-within-sections = true [tool.pytest.ini_options] asyncio_mode = "auto" diff --git a/query_exporter/config.py b/query_exporter/config.py index abf000f..9db1234 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -11,8 +11,8 @@ from pathlib import Path import re from typing import ( - Any, IO, + Any, ) from urllib.parse import ( quote_plus, diff --git a/query_exporter/db.py b/query_exporter/db.py index fa7e07a..2c791e0 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -17,8 +17,8 @@ from types import TracebackType from typing import ( Any, - cast, NamedTuple, + cast, ) from croniter import croniter diff --git a/query_exporter/loop.py b/query_exporter/loop.py index e355a2c..7f6e54f 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -2,7 +2,7 @@ import asyncio from collections import defaultdict -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from datetime import datetime from decimal import Decimal from logging import Logger @@ -10,7 +10,6 @@ from typing import ( Any, cast, - Iterator, ) from croniter import croniter @@ -27,15 +26,15 @@ ) from .config import ( - Config, DB_ERRORS_METRIC_NAME, QUERIES_METRIC_NAME, QUERY_LATENCY_METRIC_NAME, QUERY_TIMESTAMP_METRIC_NAME, + Config, ) from .db import ( - DataBase, DATABASE_LABEL, + DataBase, DataBaseConnectError, DataBaseError, Query, diff --git a/tests/config_test.py b/tests/config_test.py index 7d2891c..3a5b0b6 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -4,13 +4,13 @@ import yaml from query_exporter.config import ( - _get_parameters_sets, - _resolve_dsn, - ConfigError, DB_ERRORS_METRIC_NAME, GLOBAL_METRICS, - load_config, QUERIES_METRIC_NAME, + ConfigError, + _get_parameters_sets, + _resolve_dsn, + load_config, ) from query_exporter.db import QueryMetric diff --git a/tests/db_test.py b/tests/db_test.py index 8b9d8a5..bbf8afc 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -9,7 +9,6 @@ from query_exporter.config import DataBaseConfig from query_exporter.db import ( - create_db_engine, DataBase, DataBaseConnectError, DataBaseError, @@ -23,6 +22,7 @@ QueryMetric, QueryResults, QueryTimeoutExpired, + create_db_engine, ) diff --git a/tox.ini b/tox.ini index 42c5da7..7521884 100644 --- a/tox.ini +++ b/tox.ini @@ -25,27 +25,21 @@ commands = [testenv:format] deps = - black - isort pyproject-fmt + ruff tox-ini-fmt commands = - isort {[base]lint_files} - black -q {[base]lint_files} + ruff format {[base]lint_files} + ruff check --fix {[base]lint_files} - pyproject-fmt pyproject.toml - tox-ini-fmt tox.ini [testenv:lint] deps = - black - flake8 - flake8-pyproject - isort pyproject-fmt + ruff commands = - isort --check-only --diff {[base]lint_files} - black --check {[base]lint_files} - flake8 {[base]lint_files} + ruff check {[base]lint_files} pyproject-fmt --check pyproject.toml [testenv:run] From 9b296f819f442d787b4a1741b134fc7cae4a70d4 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 29 Oct 2024 19:22:12 +0100 Subject: [PATCH 035/110] drop unnecessary asyncio test markers --- tests/db_test.py | 21 --------------------- tests/loop_test.py | 1 - 2 files changed, 22 deletions(-) diff --git a/tests/db_test.py b/tests/db_test.py index bbf8afc..2fb6680 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -302,7 +302,6 @@ def test_results_wrong_names_with_labels(self): class TestQueryResults: - @pytest.mark.asyncio async def test_from_results(self): """The from_results method creates a QueryResult.""" engine = create_engine("sqlite://", strategy=ASYNCIO_STRATEGY) @@ -314,7 +313,6 @@ async def test_from_results(self): assert query_results.latency is None assert query_results.timestamp < time.time() - @pytest.mark.asyncio async def test_from_results_with_latency(self): """The from_results method creates a QueryResult.""" engine = create_engine("sqlite://", strategy=ASYNCIO_STRATEGY) @@ -351,7 +349,6 @@ def test_instantiate(self, db_config): assert db.config is db_config assert db.logger == logging.getLogger() - @pytest.mark.asyncio async def test_as_context_manager(self, db): """The database can be used as an async context manager.""" async with db: @@ -360,7 +357,6 @@ async def test_as_context_manager(self, db): # the db is closed at context exit assert not db.connected - @pytest.mark.asyncio async def test_connect(self, caplog, db): """The connect connects to the database.""" with caplog.at_level(logging.DEBUG): @@ -368,14 +364,12 @@ async def test_connect(self, caplog, db): assert isinstance(db._conn, AsyncConnection) assert caplog.messages == ['connected to database "db"'] - @pytest.mark.asyncio async def test_connect_lock(self, caplog, db): """The connect method has a lock to prevent concurrent calls.""" with caplog.at_level(logging.DEBUG): await asyncio.gather(db.connect(), db.connect()) assert caplog.messages == ['connected to database "db"'] - @pytest.mark.asyncio async def test_connect_error(self): """A DataBaseConnectError is raised if database connection fails.""" config = DataBaseConfig(name="db", dsn="sqlite:////invalid") @@ -384,7 +378,6 @@ async def test_connect_error(self): await db.connect() assert "unable to open database file" in str(error.value) - @pytest.mark.asyncio async def test_connect_sql(self): """If connect_sql is specified, it's run at connection.""" config = DataBaseConfig( @@ -404,7 +397,6 @@ async def execute_sql(sql): assert queries == ["SELECT 1", "SELECT 2"] await db.close() - @pytest.mark.asyncio async def test_connect_sql_fail(self, caplog): """If the SQL at connection fails, an error is raised.""" config = DataBaseConfig( @@ -422,7 +414,6 @@ async def test_connect_sql_fail(self, caplog): assert 'failed executing query "WRONG"' in str(error.value) assert 'disconnected from database "db"' in caplog.messages - @pytest.mark.asyncio async def test_close(self, caplog, db): """The close method closes database connection.""" await db.connect() @@ -433,7 +424,6 @@ async def test_close(self, caplog, db): assert connection.closed assert db._conn is None - @pytest.mark.asyncio async def test_execute_log(self, db, caplog): """A message is logged about the query being executed.""" query = Query( @@ -449,7 +439,6 @@ async def test_execute_log(self, db, caplog): await db.close() @pytest.mark.parametrize("connected", [True, False]) - @pytest.mark.asyncio async def test_execute_keep_connected(self, mocker, connected): """If keep_connected is set to true, the db is not closed.""" config = DataBaseConfig( @@ -472,7 +461,6 @@ async def test_execute_keep_connected(self, mocker, connected): mock_conn_detach.assert_called_once() await db.close() - @pytest.mark.asyncio async def test_execute_no_keep_disconnect_after_pending_queries(self): """The db is disconnected only after pending queries are run.""" config = DataBaseConfig( @@ -495,7 +483,6 @@ async def test_execute_no_keep_disconnect_after_pending_queries(self): await asyncio.gather(db.execute(query1), db.execute(query2)) assert not db.connected - @pytest.mark.asyncio async def test_execute_not_connected(self, db): """The execute recconnects to the database if not connected.""" query = Query( @@ -506,7 +493,6 @@ async def test_execute_not_connected(self, db): # the connection is kept for reuse assert not db._conn.closed - @pytest.mark.asyncio async def test_execute(self, db): """The execute method executes a query.""" sql = ( @@ -529,7 +515,6 @@ async def test_execute(self, db): ] assert isinstance(metric_results.latency, float) - @pytest.mark.asyncio async def test_execute_with_labels(self, db): """The execute method executes a query with labels.""" sql = """ @@ -559,7 +544,6 @@ async def test_execute_with_labels(self, db): MetricResult("metric2", 33, {"label2": "baz"}), ] - @pytest.mark.asyncio async def test_execute_query_invalid_count(self, caplog, db): """If the number of fields don't match, an error is raised.""" query = Query( @@ -584,7 +568,6 @@ async def test_execute_query_invalid_count(self, caplog, db): "Wrong result count from query: expected 1, got 2" ] - @pytest.mark.asyncio async def test_execute_query_invalid_count_with_labels(self, db): """If the number of fields don't match, an error is raised.""" query = Query( @@ -602,7 +585,6 @@ async def test_execute_query_invalid_count_with_labels(self, db): ) assert error.value.fatal - @pytest.mark.asyncio async def test_execute_query_invalid_names_with_labels(self, db): """If the names of fields don't match, an error is raised.""" query = Query( @@ -620,7 +602,6 @@ async def test_execute_query_invalid_names_with_labels(self, db): ) assert error.value.fatal - @pytest.mark.asyncio async def test_execute_query_traceback_debug(self, caplog, mocker, db): """Traceback are logged as debug messages.""" query = Query( @@ -646,7 +627,6 @@ async def test_execute_query_traceback_debug(self, caplog, mocker, db): # traceback is included in messages assert "await self._execute_query(query)" in caplog.messages[-1] - @pytest.mark.asyncio async def test_execute_timeout(self, caplog, db): """If the query times out, an error is raised and logged.""" query = Query( @@ -676,7 +656,6 @@ async def execute(sql, parameters): 'Execution for query "query" expired after 0.1 seconds' ] - @pytest.mark.asyncio async def test_execute_sql(self, db): """It's possible to execute raw SQL.""" await db.connect() diff --git a/tests/loop_test.py b/tests/loop_test.py index 58162eb..c30113e 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -148,7 +148,6 @@ def test_expire_series(self, metrics_expiration): } -@pytest.mark.asyncio class TestQueryLoop: async def test_start(self, query_loop): """The start method starts timed calls for queries.""" From 6734c13c78d0c7c7647ed89f293d9568a305538c Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 29 Oct 2024 19:26:14 +0100 Subject: [PATCH 036/110] snap: update to core24 --- snap/snapcraft.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ec5d2ed..8935a93 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -34,7 +34,7 @@ issues: https://github.com/albertodonato/query-exporter/issues confinement: strict grade: stable -base: core22 +base: core24 apps: daemon: @@ -73,7 +73,7 @@ parts: - unixodbc-dev stage-packages: - libmysqlclient21 - - libodbc1 + - libodbc2 - libpq5 - libxml2 prime: From e69372c17860b9d20f28a061f27c6ec90302c248 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 30 Oct 2024 12:59:57 +0100 Subject: [PATCH 037/110] snap: specify build architectures --- snap/snapcraft.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 8935a93..57e6044 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -35,6 +35,11 @@ issues: https://github.com/albertodonato/query-exporter/issues confinement: strict grade: stable base: core24 +platforms: + amd64: + arm64: + s390x: + ppc64el: apps: daemon: From ec9893acf383a27fb668823822c15cce49c0fada Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 30 Oct 2024 13:05:59 +0100 Subject: [PATCH 038/110] require Python 3.11 --- .github/workflows/ci.yml | 2 -- pyproject.toml | 3 +-- query_exporter/db.py | 7 ++++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 688b58f..46ffa78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,6 @@ jobs: strategy: matrix: python-version: - - "3.10" - "3.11" - "3.12" - "3.13" @@ -59,7 +58,6 @@ jobs: strategy: matrix: python-version: - - "3.10" - "3.11" - "3.12" - "3.13" diff --git a/pyproject.toml b/pyproject.toml index 84b903a..64ed3f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ maintainers = [ authors = [ { name = "Alberto Donato", email = "alberto.donato@gmail.com" }, ] -requires-python = ">=3.10" +requires-python = ">=3.11" classifiers = [ "Development Status :: 5 - Production/Stable", "Framework :: AsyncIO", @@ -29,7 +29,6 @@ classifiers = [ "License :: OSI Approved :: GNU General Public License (GPL)", "Operating System :: OS Independent", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/query_exporter/db.py b/query_exporter/db.py index 2c791e0..7406818 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -18,6 +18,7 @@ from typing import ( Any, NamedTuple, + Self, cast, ) @@ -165,7 +166,7 @@ class QueryResults(NamedTuple): latency: float | None = None @classmethod - async def from_results(cls, results: AsyncResultProxy) -> "QueryResults": + async def from_results(cls, results: AsyncResultProxy) -> Self: """Return a QueryResults from results for a query.""" timestamp = time() conn_info = results._result_proxy.connection.info @@ -298,7 +299,7 @@ def __init__( self._setup_query_latency_tracking() - async def __aenter__(self) -> "DataBase": + async def __aenter__(self) -> Self: await self.connect() return self @@ -352,7 +353,7 @@ async def execute(self, query: Query) -> MetricResults: try: result = await self._execute_query(query) return query.results(await QueryResults.from_results(result)) - except asyncio.TimeoutError: + except TimeoutError: raise self._query_timeout_error( query.name, cast(QueryTimeout, query.timeout) ) From 6bf0bb41c357911051921533e3f039b16e1ec786 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 2 Nov 2024 12:45:34 +0100 Subject: [PATCH 039/110] switch to SQLAlchemy 2 (#198) Drop dependency on sqlalchemy-aio, replace with local worker thread implementation. --- pyproject.toml | 3 +- query_exporter/db.py | 236 ++++++++++++++++++++++++++++++++--------- query_exporter/loop.py | 40 ++++--- tests/conftest.py | 21 ++++ tests/db_test.py | 209 +++++++++++++++++++++++++++++------- tests/loop_test.py | 138 +++++++++++++----------- 6 files changed, 472 insertions(+), 175 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 64ed3f8..f0b00b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,8 +48,7 @@ dependencies = [ "prometheus-client", "python-dateutil", "pyyaml", - "sqlalchemy<1.4", - "sqlalchemy-aio>=0.17", + "sqlalchemy>=2", "toolrack>=4", ] optional-dependencies.testing = [ diff --git a/query_exporter/db.py b/query_exporter/db.py index 7406818..a168684 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -1,14 +1,24 @@ """Database wrapper.""" import asyncio -from collections.abc import Iterable +from collections.abc import ( + Callable, + Iterable, + Sequence, +) +from concurrent import futures from dataclasses import ( dataclass, field, ) +from functools import partial from itertools import chain import logging import sys +from threading import ( + Thread, + current_thread, +) from time import ( perf_counter, time, @@ -28,16 +38,17 @@ event, text, ) +from sqlalchemy.engine import ( + Connection, + CursorResult, + Engine, + Row, +) from sqlalchemy.exc import ( ArgumentError, NoSuchModuleError, ) -from sqlalchemy_aio import ASYNCIO_STRATEGY -from sqlalchemy_aio.asyncio import AsyncioEngine -from sqlalchemy_aio.base import ( - AsyncConnection, - AsyncResultProxy, -) +from sqlalchemy.sql.elements import TextClause #: Timeout for a query QueryTimeout = int | float @@ -116,7 +127,7 @@ def __init__(self, query_name: str, message: str) -> None: ) -# database errors that mean the query won't ever succeed. Not all possible +# Database errors that mean the query won't ever succeed. Not all possible # fatal errors are tracked here, because some DBAPI errors can happen in # circumstances which can be fatal or not. Since there doesn't seem to be a # reliable way to know, there might be cases when a query will never succeed @@ -140,7 +151,7 @@ def __post_init__(self) -> None: create_db_engine(self.dsn) -def create_db_engine(dsn: str, **kwargs: Any) -> AsyncioEngine: +def create_db_engine(dsn: str, **kwargs: Any) -> Engine: """Create the database engine, validating the DSN""" try: return create_engine(dsn, **kwargs) @@ -161,22 +172,20 @@ class QueryResults(NamedTuple): """Results of a database query.""" keys: list[str] - rows: list[tuple[Any]] + rows: Sequence[Row[Any]] timestamp: float | None = None latency: float | None = None @classmethod - async def from_results(cls, results: AsyncResultProxy) -> Self: + def from_result(cls, result: CursorResult[Any]) -> Self: """Return a QueryResults from results for a query.""" timestamp = time() - conn_info = results._result_proxy.connection.info - latency = conn_info.get("query_latency", None) - return cls( - await results.keys(), - await results.fetchall(), - timestamp=timestamp, - latency=latency, - ) + keys: list[str] = [] + rows: Sequence[Row[Any]] = [] + if result.returns_rows: + keys, rows = list(result.keys()), result.all() + latency = result.connection.info.get("query_latency", None) + return cls(keys, rows, timestamp=timestamp, latency=latency) class MetricResult(NamedTuple): @@ -276,11 +285,149 @@ def _check_query_parameters(self) -> None: raise InvalidQueryParameters(self.name) +class WorkerAction: + """An action to be called in the worker thread.""" + + def __init__( + self, func: Callable[..., Any], *args: Any, **kwargs: Any + ) -> None: + self._func = partial(func, *args, **kwargs) + self._loop = asyncio.get_event_loop() + self._future = self._loop.create_future() + + def __str__(self) -> str: + return self._func.func.__name__ + + def __call__(self) -> None: + """Call the action asynchronously in a thread-safe way.""" + try: + result = self._func() + except Exception as e: + self._call_threadsafe(self._future.set_exception, e) + else: + self._call_threadsafe(self._future.set_result, result) + + async def result(self) -> Any: + """Wait for completion and return the action result.""" + return await self._future + + def _call_threadsafe(self, call: Callable[..., Any], *args: Any) -> None: + self._loop.call_soon_threadsafe(partial(call, *args)) + + +class DataBaseConnection: + """A connection to a database engine.""" + + _conn: Connection | None = None + _worker: Thread | None = None + + def __init__( + self, + dbname: str, + engine: Engine, + logger: logging.Logger = logging.getLogger(), + ) -> None: + self.dbname = dbname + self.engine = engine + self.logger = logger + self._loop = asyncio.get_event_loop() + self._queue: asyncio.Queue[WorkerAction] = asyncio.Queue() + + @property + def connected(self) -> bool: + """Whether the connection is open.""" + return self._conn is not None + + async def open(self) -> None: + """Open the connection.""" + if self.connected: + return + + self._create_worker() + await self._call_in_thread(self._connect) + + async def close(self) -> None: + """Close the connection.""" + if not self.connected: + return + + await self._call_in_thread(self._close) + self._terminate_worker() + + async def execute( + self, + sql: TextClause, + parameters: dict[str, Any] | None = None, + ) -> QueryResults: + """Execute a query, returning results.""" + if parameters is None: + parameters = {} + result = await self._call_in_thread(self._execute, sql, parameters) + query_results: QueryResults = await self._call_in_thread( + QueryResults.from_result, result + ) + return query_results + + def _create_worker(self) -> None: + assert not self._worker + self._worker = Thread( + target=self._run, name=f"DataBase-{self.dbname}", daemon=True + ) + self._worker.start() + + def _terminate_worker(self) -> None: + assert self._worker + self._worker.join() + self._worker = None + + def _connect(self) -> None: + self._conn = self.engine.connect() + + def _execute( + self, sql: TextClause, parameters: dict[str, Any] + ) -> CursorResult[Any]: + assert self._conn + return self._conn.execute(sql, parameters) + + def _close(self) -> None: + assert self._conn + self._conn.detach() + self._conn.close() + self._conn = None + + def _run(self) -> None: + """The worker thread function.""" + + def debug(message: str) -> None: + self.logger.debug(f'worker "{current_thread().name}": {message}') + + debug(f"started with ID {current_thread().native_id}") + while True: + future = asyncio.run_coroutine_threadsafe( + self._queue.get(), self._loop + ) + action = future.result() + debug(f'received action "{action}"') + action() + self._loop.call_soon_threadsafe(self._queue.task_done) + if self._conn is None: + # the connection has been closed, exit the thread + debug("shutting down") + return + + async def _call_in_thread( + self, func: Callable[..., Any], *args: Any, **kwargs: Any + ) -> Any: + """Call a sync action in the worker thread.""" + call = WorkerAction(func, *args, **kwargs) + await self._queue.put(call) + return await call.result() + + class DataBase: """A database to perform Queries.""" - _engine: AsyncioEngine - _conn: AsyncConnection | None = None + _conn: DataBaseConnection _pending_queries: int = 0 def __init__( @@ -291,27 +438,32 @@ def __init__( self.config = config self.logger = logger self._connect_lock = asyncio.Lock() - self._engine = create_db_engine( + execution_options = {} + if self.config.autocommit: + execution_options["isolation_level"] = "AUTOCOMMIT" + engine = create_db_engine( self.config.dsn, - strategy=ASYNCIO_STRATEGY, - execution_options={"autocommit": self.config.autocommit}, + execution_options=execution_options, ) - - self._setup_query_latency_tracking() + self._conn = DataBaseConnection(self.config.name, engine, self.logger) + self._setup_query_latency_tracking(engine) async def __aenter__(self) -> Self: await self.connect() return self async def __aexit__( - self, exc_type: type, exc_value: Exception, traceback: TracebackType + self, + exc_type: type, + exc_value: Exception, + traceback: TracebackType, ) -> None: await self.close() @property def connected(self) -> bool: """Whether the database is connected.""" - return self._conn is not None + return self._conn.connected async def connect(self) -> None: """Connect to the database.""" @@ -320,7 +472,7 @@ async def connect(self) -> None: return try: - self._conn = await self._engine.connect() + await self._conn.open() except Exception as error: raise self._db_error(error, exc_class=DataBaseConnectError) @@ -349,10 +501,11 @@ async def execute(self, query: Query) -> MetricResults: f'running query "{query.name}" on database "{self.config.name}"' ) self._pending_queries += 1 - self._conn: AsyncConnection try: - result = await self._execute_query(query) - return query.results(await QueryResults.from_results(result)) + query_results = await self.execute_sql( + query.sql, parameters=query.parameters, timeout=query.timeout + ) + return query.results(query_results) except TimeoutError: raise self._query_timeout_error( query.name, cast(QueryTimeout, query.timeout) @@ -372,34 +525,19 @@ async def execute_sql( sql: str, parameters: dict[str, Any] | None = None, timeout: QueryTimeout | None = None, - ) -> AsyncResultProxy: + ) -> QueryResults: """Execute a raw SQL query.""" - if parameters is None: - parameters = {} - self._conn: AsyncConnection return await asyncio.wait_for( self._conn.execute(text(sql), parameters), timeout=timeout, ) - async def _execute_query(self, query: Query) -> AsyncResultProxy: - """Execute a query.""" - return await self.execute_sql( - query.sql, parameters=query.parameters, timeout=query.timeout - ) - async def _close(self) -> None: # ensure the connection with the DB is actually closed - self._conn: AsyncConnection - self._conn.sync_connection.detach() await self._conn.close() - self._conn = None - self._pending_queries = 0 self.logger.debug(f'disconnected from database "{self.config.name}"') - def _setup_query_latency_tracking(self) -> None: - engine = self._engine.sync_engine - + def _setup_query_latency_tracking(self, engine: Engine) -> None: @event.listens_for(engine, "before_cursor_execute") # type: ignore def before_cursor_execute( conn, cursor, statement, parameters, context, executemany diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 7f6e54f..75b025b 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -83,15 +83,20 @@ def expire_series( expired = {} for name, metric_last_seen in self._last_seen.items(): expiration = cast(int, self._expirations[name]) - expired[name] = [ + expired_labels = [ label_values for label_values, last_seen in metric_last_seen.items() if timestamp > last_seen + expiration ] + if expired_labels: + expired[name] = expired_labels + # clear expired series from tracking for name, series_labels in expired.items(): for label_values in series_labels: del self._last_seen[name][label_values] + if not self._last_seen[name]: + del self._last_seen[name] return expired @@ -137,7 +142,7 @@ async def start(self) -> None: call: TimedCall if query.interval: call = PeriodicCall(self._run_query, query) - call.start(query.interval) + call.start(query.interval, now=True) elif query.schedule is not None: call = TimedCall(self._run_query, query) call.start(self._loop_times_iter(query.schedule)) @@ -153,7 +158,7 @@ async def stop(self) -> None: def clear_expired_series(self) -> None: """Clear metric series that have expired.""" - expired_series = self._last_seen.expire_series(self._timestamp()) + expired_series = self._last_seen.expire_series(self._loop.time()) for name, label_values in expired_series.items(): metric = self._registry.get_metric(name) for values in label_values: @@ -170,7 +175,7 @@ async def run_aperiodic_queries(self) -> None: def _loop_times_iter(self, schedule: str) -> Iterator[float | int]: """Wrap a croniter iterator to sync time with the loop clock.""" - cron_iter = croniter(schedule, self._now()) + cron_iter = croniter(schedule, datetime.now(gettz())) while True: cc = next(cron_iter) t = time.time() @@ -198,7 +203,7 @@ async def _execute_query(self, query: Query, dbname: str) -> None: self._increment_queries_count(db, query, "error") if error.fatal: self._logger.debug( - f'removing doomed query "{query.name}" ' + f'removing failed query "{query.name}" ' f'for database "{dbname}"' ) self._doomed_queries[query.name].add(dbname) @@ -264,20 +269,21 @@ def _update_metric( ) metric = self._registry.get_metric(name, labels=all_labels) self._update_metric_value(metric, method, value) - self._last_seen.update(name, all_labels, self._timestamp()) + self._last_seen.update(name, all_labels, self._loop.time()) def _get_metric_method(self, metric: MetricConfig) -> str: - method = { - "counter": "inc", - "gauge": "set", - "histogram": "observe", - "summary": "observe", - "enum": "state", - }[metric.type] if metric.type == "counter" and not metric.config.get( "increment", True ): method = "set" + else: + method = { + "counter": "inc", + "gauge": "set", + "histogram": "observe", + "summary": "observe", + "enum": "state", + }[metric.type] return method def _update_metric_value( @@ -325,11 +331,3 @@ def _update_query_timestamp_metric( timestamp, labels={"query": query.config_name}, ) - - def _now(self) -> datetime: - """Return the current time with local timezone.""" - return datetime.now().replace(tzinfo=gettz()) - - def _timestamp(self) -> float: - """Return the current timestamp.""" - return self._now().timestamp() diff --git a/tests/conftest.py b/tests/conftest.py index 43b6c1f..9eb582c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,6 @@ import asyncio +from collections.abc import Iterator +import re import pytest from toolrack.testing.fixtures import advance_time @@ -58,3 +60,22 @@ async def execute(self, query): mocker.patch.object(DataBase, "execute", execute) yield tracker + + +class AssertRegexpMatch: + """Assert that comparison matches the specified regexp.""" + + def __init__(self, pattern: str, flags: int = 0) -> None: + self._re = re.compile(pattern, flags) + + def __eq__(self, string: str) -> bool: + return bool(self._re.match(string)) + + def __repr__(self) -> str: + return self._re.pattern # pragma: nocover + + +@pytest.fixture +def re_match() -> Iterator[type[AssertRegexpMatch]]: + """Matcher for asserting that a string matches a regexp.""" + yield AssertRegexpMatch diff --git a/tests/db_test.py b/tests/db_test.py index 2fb6680..ae0d504 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -1,16 +1,23 @@ import asyncio +from collections.abc import Iterator import logging import time import pytest -from sqlalchemy import create_engine -from sqlalchemy_aio import ASYNCIO_STRATEGY -from sqlalchemy_aio.base import AsyncConnection +from sqlalchemy import ( + create_engine, + text, +) +from sqlalchemy.engine import ( + Connection, + Engine, +) from query_exporter.config import DataBaseConfig from query_exporter.db import ( DataBase, DataBaseConnectError, + DataBaseConnection, DataBaseError, DataBaseQueryError, InvalidQueryParameters, @@ -22,6 +29,7 @@ QueryMetric, QueryResults, QueryTimeoutExpired, + WorkerAction, create_db_engine, ) @@ -302,31 +310,118 @@ def test_results_wrong_names_with_labels(self): class TestQueryResults: - async def test_from_results(self): - """The from_results method creates a QueryResult.""" - engine = create_engine("sqlite://", strategy=ASYNCIO_STRATEGY) - async with engine.connect() as conn: - result = await conn.execute("SELECT 1 AS a, 2 AS b") - query_results = await QueryResults.from_results(result) + def test_from_result(self): + """The from_result method returns a QueryResult.""" + engine = create_engine("sqlite://") + with engine.connect() as conn: + result = conn.execute(text("SELECT 1 AS a, 2 AS b")) + query_results = QueryResults.from_result(result) assert query_results.keys == ["a", "b"] assert query_results.rows == [(1, 2)] assert query_results.latency is None assert query_results.timestamp < time.time() - async def test_from_results_with_latency(self): - """The from_results method creates a QueryResult.""" - engine = create_engine("sqlite://", strategy=ASYNCIO_STRATEGY) - async with engine.connect() as conn: - result = await conn.execute("SELECT 1 AS a, 2 AS b") + def test_from_empty(self): + """The from_result method returns empty QueryResult.""" + engine = create_engine("sqlite://") + with engine.connect() as conn: + result = conn.execute(text("PRAGMA auto_vacuum = 1")) + query_results = QueryResults.from_result(result) + assert query_results.keys == [] + assert query_results.rows == [] + assert query_results.latency is None + + def test_from_result_with_latency(self): + """The from_result method tracks call latency.""" + engine = create_engine("sqlite://") + with engine.connect() as conn: + result = conn.execute(text("SELECT 1 AS a, 2 AS b")) # simulate latency tracking - conn.sync_connection.info["query_latency"] = 1.2 - query_results = await QueryResults.from_results(result) + conn.info["query_latency"] = 1.2 + query_results = QueryResults.from_result(result) assert query_results.keys == ["a", "b"] assert query_results.rows == [(1, 2)] assert query_results.latency == 1.2 assert query_results.timestamp < time.time() +@pytest.fixture +async def conn() -> Iterator[DataBaseConnection]: + engine = create_engine("sqlite://") + connection = DataBaseConnection("db", engine) + yield connection + await connection.close() + + +class TestWorkerAction: + async def test_call_wait(self): + def func(a: int, b: int) -> int: + return a + b + + action = WorkerAction(func, 10, 20) + action() + assert await action.result() == 30 + + async def test_call_exception(self): + def func() -> None: + raise Exception("fail!") + + action = WorkerAction(func) + action() + with pytest.raises(Exception) as error: + await action.result() + assert str(error.value) == "fail!" + + +class TestDataBaseConnection: + def test_engine(self, conn): + """The connection keeps the SQLAlchemy engine.""" + assert isinstance(conn.engine, Engine) + + async def test_open(self, conn: DataBaseConnection) -> None: + """The open method opens the database connection.""" + await conn.open() + assert conn.connected + assert conn._conn is not None + assert conn._worker.is_alive() + + async def test_open_noop(self, conn: DataBaseConnection) -> None: + """The open method is a no-op if connection is already open.""" + await conn.open() + await conn.open() + assert conn.connected + + async def test_close(self, conn: DataBaseConnection) -> None: + """The close method closes the connection.""" + await conn.open() + await conn.close() + assert not conn.connected + assert conn._conn is None + + async def test_close_noop(self, conn: DataBaseConnection) -> None: + """The close method is a no-op if connection is already closed.""" + await conn.open() + await conn.close() + await conn.close() + assert not conn.connected + + async def test_execute(self, conn: DataBaseConnection) -> None: + """The connection can execute queries.""" + await conn.open() + query_results = await conn.execute(text("SELECT 1 AS a, 2 AS b")) + assert query_results.keys == ["a", "b"] + assert query_results.rows == [(1, 2)] + + async def test_execute_with_params(self, conn: DataBaseConnection) -> None: + """The connection can execute queries with parameters.""" + await conn.open() + query_results = await conn.execute( + text("SELECT :a AS a, :b AS b"), parameters={"a": 1, "b": 2} + ) + assert query_results.keys == ["a", "b"] + assert query_results.rows == [(1, 2)] + + @pytest.fixture def db_config(): return DataBaseConfig( @@ -352,23 +447,32 @@ def test_instantiate(self, db_config): async def test_as_context_manager(self, db): """The database can be used as an async context manager.""" async with db: - result = await db.execute_sql("SELECT 10 AS a, 20 AS b") - assert await result.fetchall() == [(10, 20)] + query_result = await db.execute_sql("SELECT 10 AS a, 20 AS b") + assert query_result.rows == [(10, 20)] # the db is closed at context exit assert not db.connected - async def test_connect(self, caplog, db): + async def test_connect(self, caplog, re_match, db): """The connect connects to the database.""" with caplog.at_level(logging.DEBUG): await db.connect() - assert isinstance(db._conn, AsyncConnection) - assert caplog.messages == ['connected to database "db"'] + assert db.connected + assert isinstance(db._conn._conn, Connection) + assert caplog.messages == [ + re_match(r'worker "DataBase-db": started'), + 'worker "DataBase-db": received action "_connect"', + 'connected to database "db"', + ] - async def test_connect_lock(self, caplog, db): + async def test_connect_lock(self, caplog, re_match, db): """The connect method has a lock to prevent concurrent calls.""" with caplog.at_level(logging.DEBUG): await asyncio.gather(db.connect(), db.connect()) - assert caplog.messages == ['connected to database "db"'] + assert caplog.messages == [ + re_match(r'worker "DataBase-db": started'), + 'worker "DataBase-db": received action "_connect"', + 'connected to database "db"', + ] async def test_connect_error(self): """A DataBaseConnectError is raised if database connection fails.""" @@ -414,15 +518,18 @@ async def test_connect_sql_fail(self, caplog): assert 'failed executing query "WRONG"' in str(error.value) assert 'disconnected from database "db"' in caplog.messages - async def test_close(self, caplog, db): + async def test_close(self, caplog, re_match, db): """The close method closes database connection.""" await db.connect() - connection = db._conn with caplog.at_level(logging.DEBUG): await db.close() - assert caplog.messages == ['disconnected from database "db"'] - assert connection.closed - assert db._conn is None + assert caplog.messages == [ + 'worker "DataBase-db": received action "_close"', + 'worker "DataBase-db": shutting down', + 'disconnected from database "db"', + ] + assert not db.connected + assert db._conn._conn is None async def test_execute_log(self, db, caplog): """A message is logged about the query being executed.""" @@ -435,7 +542,11 @@ async def test_execute_log(self, db, caplog): await db.connect() with caplog.at_level(logging.DEBUG): await db.execute(query) - assert caplog.messages == ['running query "query" on database "db"'] + assert caplog.messages == [ + 'running query "query" on database "db"', + 'worker "DataBase-db": received action "_execute"', + 'worker "DataBase-db": received action "from_result"', + ] await db.close() @pytest.mark.parametrize("connected", [True, False]) @@ -452,9 +563,7 @@ async def test_execute_keep_connected(self, mocker, connected): "SELECT 1.0 AS metric", ) await db.connect() - mock_conn_detach = mocker.patch.object( - db._conn.sync_connection, "detach" - ) + mock_conn_detach = mocker.patch.object(db._conn._conn, "detach") await db.execute(query) assert db.connected == connected if not connected: @@ -491,7 +600,7 @@ async def test_execute_not_connected(self, db): metric_results = await db.execute(query) assert metric_results.results == [MetricResult("metric", 1, {})] # the connection is kept for reuse - assert not db._conn.closed + assert db.connected async def test_execute(self, db): """The execute method executes a query.""" @@ -544,6 +653,14 @@ async def test_execute_with_labels(self, db): MetricResult("metric2", 33, {"label2": "baz"}), ] + async def test_execute_fail(self, caplog, db): + """If the query fails, an exception is raised.""" + query = Query("query", 10, [QueryMetric("metric", [])], "WRONG") + await db.connect() + with pytest.raises(DataBaseQueryError) as error: + await db.execute(query) + assert "syntax error" in str(error.value) + async def test_execute_query_invalid_count(self, caplog, db): """If the number of fields don't match, an error is raised.""" query = Query( @@ -585,7 +702,7 @@ async def test_execute_query_invalid_count_with_labels(self, db): ) assert error.value.fatal - async def test_execute_query_invalid_names_with_labels(self, db): + async def test_execute_invalid_names_with_labels(self, db): """If the names of fields don't match, an error is raised.""" query = Query( "query", @@ -602,7 +719,7 @@ async def test_execute_query_invalid_names_with_labels(self, db): ) assert error.value.fatal - async def test_execute_query_traceback_debug(self, caplog, mocker, db): + async def test_execute_traceback_debug(self, caplog, mocker, db): """Traceback are logged as debug messages.""" query = Query( "query", @@ -610,10 +727,8 @@ async def test_execute_query_traceback_debug(self, caplog, mocker, db): [QueryMetric("metric", [])], "SELECT 1 AS metric", ) - mocker.patch.object(db, "_execute_query").side_effect = Exception( - "boom!" - ) await db.connect() + mocker.patch.object(db, "execute_sql").side_effect = Exception("boom!") with ( caplog.at_level(logging.DEBUG), pytest.raises(DataBaseQueryError) as error, @@ -625,7 +740,7 @@ async def test_execute_query_traceback_debug(self, caplog, mocker, db): 'query "query" on database "db" failed: boom!' in caplog.messages ) # traceback is included in messages - assert "await self._execute_query(query)" in caplog.messages[-1] + assert "await self.execute_sql(" in caplog.messages[-1] async def test_execute_timeout(self, caplog, db): """If the query times out, an error is raised and logged.""" @@ -660,4 +775,18 @@ async def test_execute_sql(self, db): """It's possible to execute raw SQL.""" await db.connect() result = await db.execute_sql("SELECT 10, 20") - assert await result.fetchall() == [(10, 20)] + assert result.rows == [(10, 20)] + + @pytest.mark.parametrize( + "error,message", + [ + ("message", "message"), + (Exception("message"), "message"), + (Exception(), "Exception"), + ], + ) + def test_error_message( + self, db: DataBase, error: str | Exception, message: str + ) -> None: + """An error message is returned both for strings and exceptions.""" + assert db._error_message(error) == message diff --git a/tests/loop_test.py b/tests/loop_test.py index c30113e..a269ab2 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -1,10 +1,10 @@ import asyncio from collections import defaultdict -from datetime import datetime +from collections.abc import Callable, Iterator from decimal import Decimal import logging from pathlib import Path -import re +from typing import Any from prometheus_aioexporter import MetricsRegistry import pytest @@ -18,21 +18,8 @@ from query_exporter.db import DataBase -class re_match: - """Assert that comparison matches the specified regexp.""" - - def __init__(self, pattern, flags=0): - self._re = re.compile(pattern, flags) - - def __eq__(self, string): - return bool(self._re.match(string)) - - def __repr__(self): - return self._re.pattern # pragma: nocover - - @pytest.fixture -def config_data(): +def config_data() -> Iterator[dict[str, Any]]: yield { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "gauge"}}, @@ -48,15 +35,17 @@ def config_data(): @pytest.fixture -def registry(): +def registry() -> Iterator[MetricsRegistry]: yield MetricsRegistry() @pytest.fixture -async def make_query_loop(tmp_path, config_data, registry): +async def make_query_loop( + tmp_path: Path, config_data: dict[str, Any], registry: MetricsRegistry +) -> Iterator[Callable[[], MetricsRegistry]]: query_loops = [] - def make_loop(): + def make_loop() -> loop.QueryLoop: config_file = tmp_path / "config.yaml" config_file.write_text(yaml.dump(config_data), "utf-8") logger = logging.getLogger() @@ -75,19 +64,12 @@ def make_loop(): @pytest.fixture -async def query_loop(make_query_loop): +async def query_loop( + make_query_loop: Callable[[], loop.QueryLoop], +) -> Iterator[loop.QueryLoop]: yield make_query_loop() -@pytest.fixture -def metrics_expiration(): - yield { - "m1": 50, - "m2": 100, - "m3": None, - } - - def metric_values(metric, by_labels=()): """Return values for the metric.""" if metric._type == "gauge": @@ -107,7 +89,7 @@ def metric_values(metric, by_labels=()): return values if by_labels else values[suffix] -async def run_queries(db_file: Path, *queries: str): +async def run_queries(db_file: Path, *queries: str) -> None: config = DataBaseConfig(name="db", dsn=f"sqlite:///{db_file}") async with DataBase(config) as db: for query in queries: @@ -115,9 +97,9 @@ async def run_queries(db_file: Path, *queries: str): class TestMetricsLastSeen: - def test_update(self, metrics_expiration): + def test_update(self) -> None: """Last seen times are tracked for each series of metrics with expiration.""" - last_seen = loop.MetricsLastSeen(metrics_expiration) + last_seen = loop.MetricsLastSeen({"m1": 50, "m2": 100}) last_seen.update("m1", {"l1": "v1", "l2": "v2"}, 100) last_seen.update("m1", {"l1": "v3", "l2": "v4"}, 200) last_seen.update("other", {"l3": "v100"}, 300) @@ -128,41 +110,63 @@ def test_update(self, metrics_expiration): } } - def test_update_label_values_sorted_by_name(self, metrics_expiration): + def test_update_label_values_sorted_by_name(self) -> None: """Last values are sorted by label names.""" - last_seen = loop.MetricsLastSeen(metrics_expiration) + last_seen = loop.MetricsLastSeen({"m1": 50}) last_seen.update("m1", {"l2": "v2", "l1": "v1"}, 100) assert last_seen._last_seen == {"m1": {("v1", "v2"): 100}} - def test_expire_series(self, metrics_expiration): + def test_expire_series_not_expired(self) -> None: + """If no entry for a metric is expired, it's not returned.""" + last_seen = loop.MetricsLastSeen({"m1": 50}) + last_seen.update("m1", {"l1": "v1", "l2": "v2"}, 10) + last_seen.update("m1", {"l1": "v3", "l2": "v4"}, 20) + assert last_seen.expire_series(30) == {} + assert last_seen._last_seen == { + "m1": { + ("v1", "v2"): 10, + ("v3", "v4"): 20, + } + } + + def test_expire_series(self) -> None: """Expired metric series are returned and removed.""" - last_seen = loop.MetricsLastSeen(metrics_expiration) + last_seen = loop.MetricsLastSeen({"m1": 50, "m2": 100}) last_seen.update("m1", {"l1": "v1", "l2": "v2"}, 10) last_seen.update("m1", {"l1": "v3", "l2": "v4"}, 100) last_seen.update("m2", {"l3": "v100"}, 100) - expired = last_seen.expire_series(120) - assert expired == {"m1": [("v1", "v2")], "m2": []} + assert last_seen.expire_series(120) == {"m1": [("v1", "v2")]} assert last_seen._last_seen == { "m1": {("v3", "v4"): 100}, "m2": {("v100",): 100}, } + def test_expire_no_labels(self) -> None: + last_seen = loop.MetricsLastSeen({"m1": 50}) + last_seen.update("m1", {}, 10) + expired = last_seen.expire_series(120) + assert expired == {"m1": [()]} + assert last_seen._last_seen == {} + class TestQueryLoop: - async def test_start(self, query_loop): + async def test_start(self, query_tracker, query_loop) -> None: """The start method starts timed calls for queries.""" await query_loop.start() timed_call = query_loop._timed_calls["q"] assert timed_call.running + await query_tracker.wait_results() - async def test_stop(self, query_loop): + async def test_stop(self, query_loop) -> None: """The stop method stops timed calls for queries.""" await query_loop.start() timed_call = query_loop._timed_calls["q"] await query_loop.stop() assert not timed_call.running - async def test_run_query(self, query_tracker, query_loop, registry): + async def test_run_query( + self, query_tracker, query_loop, registry + ) -> None: """Queries are run and update metrics.""" await query_loop.start() await query_tracker.wait_results() @@ -288,14 +292,20 @@ async def test_update_metric_decimal_value( assert value == 100.123 assert isinstance(value, float) - async def test_run_query_log(self, caplog, query_tracker, query_loop): + async def test_run_query_log( + self, caplog, re_match, query_tracker, query_loop + ): """Debug messages are logged on query execution.""" caplog.set_level(logging.DEBUG) await query_loop.start() await query_tracker.wait_queries() assert caplog.messages == [ + re_match(r'worker "DataBase-db": started'), + 'worker "DataBase-db": received action "_connect"', 'connected to database "db"', 'running query "q" on database "db"', + 'worker "DataBase-db": received action "_execute"', + 'worker "DataBase-db": received action "from_result"', 'updating metric "m" set 100.0 {database="db"}', re_match( r'updating metric "query_latency" observe .* \{database="db",query="q"\}' @@ -307,7 +317,7 @@ async def test_run_query_log(self, caplog, query_tracker, query_loop): ] async def test_run_query_log_labels( - self, caplog, query_tracker, config_data, make_query_loop + self, caplog, re_match, query_tracker, config_data, make_query_loop ): """Debug messages include metric labels.""" config_data["metrics"]["m"]["labels"] = ["l"] @@ -317,8 +327,12 @@ async def test_run_query_log_labels( await query_loop.start() await query_tracker.wait_queries() assert caplog.messages == [ + re_match(r'worker "DataBase-db": started'), + 'worker "DataBase-db": received action "_connect"', 'connected to database "db"', 'running query "q" on database "db"', + 'worker "DataBase-db": received action "_execute"', + 'worker "DataBase-db": received action "from_result"', 'updating metric "m" set 100.0 {database="db",l="foo"}', re_match( r'updating metric "query_latency" observe .* \{database="db",query="q"\}' @@ -346,7 +360,7 @@ async def test_run_query_increase_database_error_count( """Count of database errors is incremented on failed connection.""" query_loop = make_query_loop() db = query_loop._databases["db"] - mock_connect = mocker.patch.object(db._engine, "connect") + mock_connect = mocker.patch.object(db._conn.engine, "connect") mock_connect.side_effect = Exception("connection failed") await query_loop.start() await query_tracker.wait_failures() @@ -403,19 +417,23 @@ async def test_run_query_at_interval( assert len(query_tracker.queries) == 2 async def test_run_timed_queries_invalid_result_count( - self, query_tracker, config_data, make_query_loop, advance_time + self, query_tracker, config_data, make_query_loop ): """Timed queries returning invalid elements count are removed.""" config_data["queries"]["q"]["sql"] = "SELECT 100.0 AS a, 200.0 AS b" + config_data["queries"]["q"]["interval"] = 1.0 query_loop = make_query_loop() await query_loop.start() - await advance_time(0) # kick the first run - assert len(query_tracker.queries) == 1 - assert len(query_tracker.results) == 0 - # the query is not run again - await advance_time(5) + timed_call = query_loop._timed_calls["q"] + await asyncio.sleep(1.1) + await query_tracker.wait_failures() + assert len(query_tracker.failures) == 1 assert len(query_tracker.results) == 0 - await advance_time(5) + # the query has been stopped and removed + assert not timed_call.running + await asyncio.sleep(1.1) + await query_tracker.wait_failures() + assert len(query_tracker.failures) == 1 assert len(query_tracker.results) == 0 async def test_run_timed_queries_invalid_result_count_stop_task( @@ -539,8 +557,8 @@ async def test_run_aperiodic_queries_not_removed_if_not_failing_on_all_dbs( async def test_clear_expired_series( self, - mocker, tmp_path, + advance_time, query_tracker, config_data, make_query_loop, @@ -555,7 +573,6 @@ async def test_clear_expired_series( "expiration": 10, } ) - # call metric collection directly config_data["queries"]["q"]["sql"] = "SELECT * FROM test" del config_data["queries"]["q"]["interval"] @@ -566,24 +583,19 @@ async def test_clear_expired_series( 'INSERT INTO test VALUES (20, "bar")', ) query_loop = make_query_loop() - mock_timestamp = mocker.patch.object(query_loop, "_timestamp") - mock_timestamp.return_value = datetime.now().timestamp() await query_loop.run_aperiodic_queries() await query_tracker.wait_results() queries_metric = registry.get_metric("m") - assert metric_values(queries_metric, by_labels=("l",)), { + assert metric_values(queries_metric, by_labels=("l",)) == { ("foo",): 10.0, ("bar",): 20.0, } - await run_queries( - db, - "DELETE FROM test WHERE m = 10", - ) - # mock that more time has passed than expiration - mock_timestamp.return_value += 20 + await run_queries(db, "DELETE FROM test WHERE m = 10") + # go beyond expiration time + await advance_time(20) await query_loop.run_aperiodic_queries() await query_tracker.wait_results() query_loop.clear_expired_series() - assert metric_values(queries_metric, by_labels=("l",)), { + assert metric_values(queries_metric, by_labels=("l",)) == { ("bar",): 20.0, } From b7447b3aff073bce70302b5cb9c57ce32c624157 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 2 Nov 2024 12:58:40 +0100 Subject: [PATCH 040/110] add typing to config tests --- query_exporter/db.py | 1 - tests/config_test.py | 341 ++++++++++++++++++++++++++++--------------- tests/conftest.py | 75 +++++----- 3 files changed, 256 insertions(+), 161 deletions(-) diff --git a/query_exporter/db.py b/query_exporter/db.py index a168684..a738ace 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -6,7 +6,6 @@ Iterable, Sequence, ) -from concurrent import futures from dataclasses import ( dataclass, field, diff --git a/tests/config_test.py b/tests/config_test.py index 3a5b0b6..c4333d5 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -1,6 +1,10 @@ +from collections.abc import Callable, Iterator import logging +from pathlib import Path +from typing import Any import pytest +from pytest import LogCaptureFixture import yaml from query_exporter.config import ( @@ -16,14 +20,14 @@ @pytest.fixture -def logger(caplog): +def logger(caplog: LogCaptureFixture) -> Iterator[logging.Logger]: with caplog.at_level("DEBUG"): yield logging.getLogger() caplog.clear() @pytest.fixture -def config_full(): +def config_full() -> Iterator[dict[str, Any]]: yield { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "gauge", "labels": ["l1", "l2"]}}, @@ -38,11 +42,14 @@ def config_full(): } +ConfigWriter = Callable[[dict[str, Any]], Path] + + @pytest.fixture -def write_config(tmp_path): +def write_config(tmp_path: Path) -> Iterator[ConfigWriter]: path = tmp_path / "config" - def write(data): + def write(data: dict[str, Any]) -> Path: path.write_text(yaml.dump(data), "utf-8") return path @@ -129,9 +136,11 @@ def write(data): class TestLoadConfig: - def test_load_databases_section(self, logger, write_config): + def test_load_databases_section( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """The 'databases' section is loaded from the config file.""" - config = { + cfg = { "databases": { "db1": {"dsn": "sqlite:///foo"}, "db2": { @@ -143,12 +152,12 @@ def test_load_databases_section(self, logger, write_config): "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: - result = load_config(fd, logger) - assert {"db1", "db2"} == set(result.databases) - database1 = result.databases["db1"] - database2 = result.databases["db2"] + config = load_config(fd, logger) + assert {"db1", "db2"} == set(config.databases) + database1 = config.databases["db1"] + database2 = config.databases["db2"] assert database1.name == "db1" assert database1.dsn == "sqlite:///foo" assert database1.keep_connected @@ -158,22 +167,30 @@ def test_load_databases_section(self, logger, write_config): assert not database2.keep_connected assert not database2.autocommit - def test_load_databases_dsn_from_env(self, logger, write_config): + def test_load_databases_dsn_from_env( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """The database DSN can be loaded from env.""" - config = { + cfg = { "databases": {"db1": {"dsn": "env:FOO"}}, "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: config = load_config(fd, logger, env={"FOO": "sqlite://"}) assert config.databases["db1"].dsn == "sqlite://" - def test_load_databases_missing_dsn(self, logger, write_config): + def test_load_databases_missing_dsn( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if the 'dsn' key is missing for a database.""" - config = {"databases": {"db1": {}}, "metrics": {}, "queries": {}} - config_file = write_config(config) + cfg: dict[str, Any] = { + "databases": {"db1": {}}, + "metrics": {}, + "queries": {}, + } + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger) assert ( @@ -181,21 +198,25 @@ def test_load_databases_missing_dsn(self, logger, write_config): == "Invalid config at databases/db1: 'dsn' is a required property" ) - def test_load_databases_invalid_dsn(self, logger, write_config): + def test_load_databases_invalid_dsn( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if the DSN is invalid.""" - config = { + cfg = { "databases": {"db1": {"dsn": "invalid"}}, "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger) assert str(err.value) == 'Invalid database DSN: "invalid"' - def test_load_databases_dsn_details(self, logger, write_config): + def test_load_databases_dsn_details( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """The DSN can be specified as a dictionary.""" - config = { + cfg = { "databases": { "db1": { "dsn": { @@ -207,16 +228,16 @@ def test_load_databases_dsn_details(self, logger, write_config): "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: config = load_config(fd, logger) assert config.databases["db1"].dsn == "sqlite:///path/to/file" def test_load_databases_dsn_details_only_dialect( - self, logger, write_config - ): + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """Only the "dialect" key is required in DSN.""" - config = { + cfg = { "databases": { "db1": { "dsn": { @@ -227,62 +248,69 @@ def test_load_databases_dsn_details_only_dialect( "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: config = load_config(fd, logger) assert config.databases["db1"].dsn == "sqlite://" - def test_load_databases_dsn_invalid_env(self, logger, write_config): + def test_load_databases_dsn_invalid_env( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if the DSN from environment is invalid.""" - config = { + cfg = { "databases": {"db1": {"dsn": "env:NOT-VALID"}}, "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger) assert str(err.value) == 'Invalid variable name: "NOT-VALID"' - def test_load_databases_dsn_undefined_env(self, logger, write_config): + def test_load_databases_dsn_undefined_env( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if the environ variable for DSN is undefined.""" - config = { + cfg = { "databases": {"db1": {"dsn": "env:FOO"}}, "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger, env={}) assert str(err.value) == 'Undefined variable: "FOO"' def test_load_databases_dsn_from_file( - self, tmp_path, logger, write_config - ): + self, + tmp_path: Path, + logger: logging.Logger, + write_config: ConfigWriter, + ) -> None: """The database DSN can be loaded from a file.""" dsn = "sqlite:///foo" dsn_path = tmp_path / "dsn" dsn_path.write_text(dsn) - config = { + cfg = { "databases": {"db1": {"dsn": f"file:{dsn_path}"}}, "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: config = load_config(fd, logger) assert config.databases["db1"].dsn == dsn def test_load_databases_dsn_from_file_not_found( - self, logger, write_config - ): + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if the DSN file can't be read.""" - config = { + cfg = { "databases": {"db1": {"dsn": "file:/not/found"}}, "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger) assert ( @@ -290,9 +318,11 @@ def test_load_databases_dsn_from_file_not_found( == 'Unable to read dsn file : "/not/found": No such file or directory' ) - def test_load_databases_no_labels(self, logger, write_config): + def test_load_databases_no_labels( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """If no labels are defined, an empty dict is returned.""" - config = { + cfg = { "databases": { "db": { "dsn": "sqlite://", @@ -301,15 +331,17 @@ def test_load_databases_no_labels(self, logger, write_config): "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: result = load_config(fd, logger) db = result.databases["db"] assert db.labels == {} - def test_load_databases_labels(self, logger, write_config): + def test_load_databases_labels( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """Labels can be defined for databases.""" - config = { + cfg = { "databases": { "db": { "dsn": "sqlite://", @@ -319,15 +351,17 @@ def test_load_databases_labels(self, logger, write_config): "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: result = load_config(fd, logger) db = result.databases["db"] assert db.labels == {"label1": "value1", "label2": "value2"} - def test_load_databases_labels_not_all_same(self, logger, write_config): + def test_load_databases_labels_not_all_same( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """If not all databases have the same labels, an error is raised.""" - config = { + cfg = { "databases": { "db1": { "dsn": "sqlite://", @@ -341,14 +375,16 @@ def test_load_databases_labels_not_all_same(self, logger, write_config): "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger, env={}) assert str(err.value) == "Not all databases define the same labels" - def test_load_databases_connect_sql(self, logger, write_config): + def test_load_databases_connect_sql( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """Databases can have queries defined to run on connection.""" - config = { + cfg = { "databases": { "db": { "dsn": "sqlite://", @@ -358,14 +394,16 @@ def test_load_databases_connect_sql(self, logger, write_config): "metrics": {}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: result = load_config(fd, logger, env={}) assert result.databases["db"].connect_sql == ["SELECT 1", "SELECT 2"] - def test_load_metrics_section(self, logger, write_config): + def test_load_metrics_section( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """The 'metrics' section is loaded from the config file.""" - config = { + cfg = { "databases": {"db1": {"dsn": "sqlite://"}}, "metrics": { "metric1": { @@ -388,7 +426,7 @@ def test_load_metrics_section(self, logger, write_config): }, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: result = load_config(fd, logger) metric1 = result.metrics["metric1"] @@ -416,14 +454,16 @@ def test_load_metrics_section(self, logger, write_config): assert result.metrics.get(DB_ERRORS_METRIC_NAME) is not None assert result.metrics.get(QUERIES_METRIC_NAME) is not None - def test_load_metrics_overlap_reserved_label(self, logger, write_config): + def test_load_metrics_overlap_reserved_label( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if reserved labels are used.""" - config = { + cfg = { "databases": {"db1": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "gauge", "labels": ["database"]}}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger) assert ( @@ -431,14 +471,16 @@ def test_load_metrics_overlap_reserved_label(self, logger, write_config): == 'Labels for metric "m" overlap with reserved/database ones: database' ) - def test_load_metrics_overlap_database_label(self, logger, write_config): + def test_load_metrics_overlap_database_label( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if database labels are used for metrics.""" - config = { + cfg = { "databases": {"db1": {"dsn": "sqlite://", "labels": {"l1": "v1"}}}, "metrics": {"m": {"type": "gauge", "labels": ["l1"]}}, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger) assert ( @@ -448,8 +490,12 @@ def test_load_metrics_overlap_database_label(self, logger, write_config): @pytest.mark.parametrize("global_name", list(GLOBAL_METRICS)) def test_load_metrics_reserved_name( - self, config_full, write_config, global_name - ): + self, + config_full: dict[str, Any], + logger: logging.Logger, + write_config: ConfigWriter, + global_name: str, + ) -> None: """An error is raised if a reserved label name is used.""" config_full["metrics"][global_name] = {"type": "counter"} config_file = write_config(config_full) @@ -460,16 +506,18 @@ def test_load_metrics_reserved_name( == f'Label name "{global_name} is reserved for builtin metric' ) - def test_load_metrics_unsupported_type(self, logger, write_config): + def test_load_metrics_unsupported_type( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if an unsupported metric type is passed.""" - config = { + cfg = { "databases": {"db1": {"dsn": "sqlite://"}}, "metrics": { "metric1": {"type": "info", "description": "info metric"} }, "queries": {}, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger) assert str(err.value) == ( @@ -477,9 +525,11 @@ def test_load_metrics_unsupported_type(self, logger, write_config): "['counter', 'enum', 'gauge', 'histogram', 'summary']" ) - def test_load_queries_section(self, logger, write_config): + def test_load_queries_section( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """The 'queries' section is loaded from the config file.""" - config = { + cfg = { "databases": { "db1": {"dsn": "sqlite:///foo"}, "db2": {"dsn": "sqlite:///bar"}, @@ -503,7 +553,7 @@ def test_load_queries_section(self, logger, write_config): }, }, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: result = load_config(fd, logger) query1 = result.queries["q1"] @@ -519,9 +569,11 @@ def test_load_queries_section(self, logger, write_config): assert query2.sql == "SELECT 2" assert query2.parameters == {} - def test_load_queries_section_with_parameters(self, logger, write_config): + def test_load_queries_section_with_parameters( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """Queries can have parameters.""" - config = { + cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary", "labels": ["l"]}}, "queries": { @@ -537,7 +589,7 @@ def test_load_queries_section_with_parameters(self, logger, write_config): }, }, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: result = load_config(fd, logger) query1 = result.queries["q[params0]"] @@ -560,10 +612,10 @@ def test_load_queries_section_with_parameters(self, logger, write_config): } def test_load_queries_section_with_parameters_matrix( - self, logger, write_config - ): + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """Queries can have parameters matrix.""" - config = { + cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary", "labels": ["l"]}}, "queries": { @@ -579,7 +631,7 @@ def test_load_queries_section_with_parameters_matrix( }, }, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: result = load_config(fd, logger) @@ -624,10 +676,10 @@ def test_load_queries_section_with_parameters_matrix( } def test_load_queries_section_with_wrong_parameters( - self, logger, write_config - ): + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if query parameters don't match.""" - config = { + cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary", "labels": ["l"]}}, "queries": { @@ -643,7 +695,7 @@ def test_load_queries_section_with_wrong_parameters( }, }, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger) assert ( @@ -652,10 +704,10 @@ def test_load_queries_section_with_wrong_parameters( ) def test_load_queries_section_with_schedule_and_interval( - self, logger, write_config - ): + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if query schedule and interval are both present.""" - config = { + cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary"}}, "queries": { @@ -668,7 +720,7 @@ def test_load_queries_section_with_schedule_and_interval( }, }, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger) assert ( @@ -676,9 +728,11 @@ def test_load_queries_section_with_schedule_and_interval( == 'Invalid schedule for query "q": both interval and schedule specified' ) - def test_load_queries_section_invalid_schedule(self, logger, write_config): + def test_load_queries_section_invalid_schedule( + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """An error is raised if query schedule has wrong format.""" - config = { + cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary"}}, "queries": { @@ -690,7 +744,7 @@ def test_load_queries_section_invalid_schedule(self, logger, write_config): }, }, } - config_file = write_config(config) + config_file = write_config(cfg) with pytest.raises(ConfigError) as err, config_file.open() as fd: load_config(fd, logger) assert ( @@ -699,8 +753,11 @@ def test_load_queries_section_invalid_schedule(self, logger, write_config): ) def test_load_queries_section_timeout( - self, logger, config_full, write_config - ): + self, + logger: logging.Logger, + config_full: dict[str, Any], + write_config: ConfigWriter, + ) -> None: """Query configuration can include a timeout.""" config_full["queries"]["q"]["timeout"] = 2.0 config_file = write_config(config_full) @@ -727,8 +784,13 @@ def test_load_queries_section_timeout( ], ) def test_load_queries_section_invalid_timeout( - self, logger, config_full, write_config, timeout, error_message - ): + self, + logger: logging.Logger, + config_full: dict[str, Any], + write_config: ConfigWriter, + timeout: float, + error_message: str, + ) -> None: """An error is raised if query timeout is invalid.""" config_full["queries"]["q"]["timeout"] = timeout config_file = write_config(config_full) @@ -772,8 +834,12 @@ def test_load_queries_section_invalid_timeout( ], ) def test_configuration_incorrect( - self, logger, write_config, config, error_message - ): + self, + logger: logging.Logger, + write_config: ConfigWriter, + config: dict[str, Any], + error_message: str, + ) -> None: """An error is raised if configuration is incorrect.""" config_file = write_config(config) with pytest.raises(ConfigError) as err, config_file.open() as fd: @@ -781,8 +847,12 @@ def test_configuration_incorrect( assert str(err.value) == error_message def test_configuration_warning_unused( - self, caplog, logger, config_full, write_config - ): + self, + caplog: LogCaptureFixture, + logger: logging.Logger, + config_full: dict[str, Any], + write_config: ConfigWriter, + ) -> None: """A warning is logged if unused entries are present in config.""" config_full["databases"]["db2"] = {"dsn": "sqlite://"} config_full["databases"]["db3"] = {"dsn": "sqlite://"} @@ -797,17 +867,17 @@ def test_configuration_warning_unused( ] def test_load_queries_missing_interval_default_to_none( - self, logger, write_config - ): + self, logger: logging.Logger, write_config: ConfigWriter + ) -> None: """If the interval is not specified, it defaults to None.""" - config = { + cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary"}}, "queries": { "q": {"databases": ["db"], "metrics": ["m"], "sql": "SELECT 1"} }, } - config_file = write_config(config) + config_file = write_config(cfg) with config_file.open() as fd: config = load_config(fd, logger) assert config.queries["q"].interval is None @@ -825,8 +895,13 @@ def test_load_queries_missing_interval_default_to_none( ], ) def test_load_queries_interval( - self, logger, config_full, write_config, interval, value - ): + self, + logger: logging.Logger, + config_full: dict[str, Any], + write_config: ConfigWriter, + interval: str | int | None, + value: int | None, + ) -> None: """The query interval can be specified with suffixes.""" config_full["queries"]["q"]["interval"] = interval config_file = write_config(config_full) @@ -835,8 +910,11 @@ def test_load_queries_interval( assert config.queries["q"].interval == value def test_load_queries_interval_not_specified( - self, logger, config_full, write_config - ): + self, + logger: logging.Logger, + config_full: dict[str, Any], + write_config: ConfigWriter, + ) -> None: """If the interval is not specified, it's set to None.""" del config_full["queries"]["q"]["interval"] config_file = write_config(config_full) @@ -846,8 +924,12 @@ def test_load_queries_interval_not_specified( @pytest.mark.parametrize("interval", ["1x", "wrong", "1.5m"]) def test_load_queries_invalid_interval_string( - self, logger, config_full, write_config, interval - ): + self, + logger: logging.Logger, + config_full: dict[str, Any], + write_config: ConfigWriter, + interval: str, + ) -> None: """An invalid string query interval raises an error.""" config_full["queries"]["q"]["interval"] = interval config_file = write_config(config_full) @@ -860,8 +942,12 @@ def test_load_queries_invalid_interval_string( @pytest.mark.parametrize("interval", [0, -20]) def test_load_queries_invalid_interval_number( - self, logger, config_full, write_config, interval - ): + self, + logger: logging.Logger, + config_full: dict[str, Any], + write_config: ConfigWriter, + interval: int, + ) -> None: """An invalid integer query interval raises an error.""" config_full["queries"]["q"]["interval"] = interval config_file = write_config(config_full) @@ -872,7 +958,12 @@ def test_load_queries_invalid_interval_number( == f"Invalid config at queries/q/interval: {interval} is less than the minimum of 1" ) - def test_load_queries_no_metrics(self, logger, config_full, write_config): + def test_load_queries_no_metrics( + self, + logger: logging.Logger, + config_full: dict[str, Any], + write_config: ConfigWriter, + ) -> None: """An error is raised if no metrics are configured.""" config_full["queries"]["q"]["metrics"] = [] config_file = write_config(config_full) @@ -884,8 +975,11 @@ def test_load_queries_no_metrics(self, logger, config_full, write_config): ) def test_load_queries_no_databases( - self, logger, config_full, write_config - ): + self, + logger: logging.Logger, + config_full: dict[str, Any], + write_config: ConfigWriter, + ) -> None: """An error is raised if no databases are configured.""" config_full["queries"]["q"]["databases"] = [] config_file = write_config(config_full) @@ -909,8 +1003,13 @@ def test_load_queries_no_databases( ], ) def test_load_metrics_expiration( - self, logger, config_full, write_config, expiration, value - ): + self, + logger: logging.Logger, + config_full: dict[str, Any], + write_config: ConfigWriter, + expiration: str | int | None, + value: int | None, + ) -> None: """The metric series expiration time can be specified with suffixes.""" config_full["metrics"]["m"]["expiration"] = expiration config_file = write_config(config_full) @@ -920,7 +1019,7 @@ def test_load_metrics_expiration( class TestResolveDSN: - def test_all_details(self): + def test_all_details(self) -> None: """The DSN can be specified as a dictionary.""" details = { "dialect": "postgresql", @@ -936,7 +1035,7 @@ def test_all_details(self): == "postgresql://user:secret@dbsever:1234/mydb?foo=bar&baz=bza" ) - def test_db_as_path(self): + def test_db_as_path(self) -> None: """If the db name is a path, it's treated accordingly.""" details = { "dialect": "sqlite", @@ -944,7 +1043,7 @@ def test_db_as_path(self): } assert _resolve_dsn(details, {}) == "sqlite:///path/to/file" - def test_encode_user_password(self): + def test_encode_user_password(self) -> None: """The user and password are encoded.""" details = { "dialect": "postgresql", @@ -958,7 +1057,7 @@ def test_encode_user_password(self): == "postgresql://us%25r:my+pass@dbsever/mydb" ) - def test_encode_options(self): + def test_encode_options(self) -> None: """Option parmaeters are encoded.""" details = { "dialect": "postgresql", @@ -975,8 +1074,8 @@ def test_encode_options(self): class TestGetParametersSets: - def test_list(self): - params = [ + def test_list(self) -> None: + params: list[dict[str, Any]] = [ { "param1": 100, "param2": "foo", @@ -988,8 +1087,8 @@ def test_list(self): ] assert list(_get_parameters_sets(params)) == params - def test_dict(self): - params = { + def test_dict(self) -> None: + params: dict[str, list[dict[str, Any]]] = { "param1": [ { "sub1": 100, diff --git a/tests/conftest.py b/tests/conftest.py index 9eb582c..b56b77c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,57 +1,54 @@ import asyncio -from collections.abc import Iterator +from collections.abc import AsyncIterator, Iterator import re import pytest +from pytest_mock import MockerFixture from toolrack.testing.fixtures import advance_time -from query_exporter.db import DataBase +from query_exporter.db import DataBase, MetricResults, Query __all__ = ["advance_time", "query_tracker"] -@pytest.fixture -def query_tracker(request, mocker, event_loop): - """Return a list collecting Query executed by DataBases.""" +class QueryTracker: + def __init__(self) -> None: + self.queries: list[Query] = [] + self.results: list[MetricResults] = [] + self.failures: list[Exception] = [] + self._loop = asyncio.get_event_loop() + + async def wait_queries(self, count: int = 1, timeout: int = 5) -> None: + await self._wait("queries", count, timeout) + + async def wait_results(self, count: int = 1, timeout: int = 5) -> None: + await self._wait("results", count, timeout) - class QueryTracker: - def __init__(self): - self.queries = [] - self.results = [] - self.failures = [] - - async def wait_queries(self, count=1, timeout=5): - await self._wait("queries", count, timeout) - - async def wait_results(self, count=1, timeout=5): - await self._wait("results", count, timeout) - - async def wait_failures(self, count=1, timeout=5): - await self._wait("failures", count, timeout) - - async def _wait(self, attr, count, timeout): - if "advance_time" in request.fixturenames: - # can't depend explicitly on the advance_time fixture or it - # would activate and always require explicit time advance - time_advance_func = request.getfixturevalue("advance_time") - else: - time_advance_func = asyncio.sleep - - end = event_loop.time() + timeout - while event_loop.time() < end: - collection = getattr(self, attr) - if len(collection) >= count: - return collection - await time_advance_func(0.05) - raise TimeoutError(f"No {attr} found after {timeout}s") + async def wait_failures(self, count: int = 1, timeout: int = 5) -> None: + await self._wait("failures", count, timeout) + async def _wait(self, attr: str, count: int, timeout: int) -> None: + end = self._loop.time() + timeout + while self._loop.time() < end: + collection = getattr(self, attr) + if len(collection) >= count: + return + await asyncio.sleep(0.05) + raise TimeoutError(f"No {attr} found after {timeout}s") + + +@pytest.fixture +async def query_tracker( + request: pytest.FixtureRequest, mocker: MockerFixture +) -> AsyncIterator[QueryTracker]: + """Return a list collecting Query executed by DataBases.""" tracker = QueryTracker() orig_execute = DataBase.execute - async def execute(self, query): + async def execute(db: DataBase, query: Query) -> MetricResults: tracker.queries.append(query) try: - result = await orig_execute(self, query) + result = await orig_execute(db, query) except Exception as e: tracker.failures.append(e) raise @@ -68,8 +65,8 @@ class AssertRegexpMatch: def __init__(self, pattern: str, flags: int = 0) -> None: self._re = re.compile(pattern, flags) - def __eq__(self, string: str) -> bool: - return bool(self._re.match(string)) + def __eq__(self, string: object) -> bool: + return bool(self._re.match(str(string))) def __repr__(self) -> str: return self._re.pattern # pragma: nocover From 52ba33599e1314d99bb5b6aa55b0eac583e4a366 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 19 Nov 2024 18:48:02 +0100 Subject: [PATCH 041/110] Version 2.11.0 --- CHANGES.rst | 9 +++++++++ query_exporter/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index eda09a8..2a6bb50 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,12 @@ +v2.11.0 - 2024-11-19 +==================== + +- Switch to SQLAlchemy 2. +- Require Python 3.11. +- [docker] Use Python 3.13 on Debian Bookwork as base image. +- [snap] Rebase on core24. + + v2.10.0 - 2024-01-28 ==================== diff --git a/query_exporter/__init__.py b/query_exporter/__init__.py index 3f0d965..59ee234 100644 --- a/query_exporter/__init__.py +++ b/query_exporter/__init__.py @@ -1,3 +1,3 @@ """Export Prometheus metrics generated from SQL queries.""" -__version__ = "2.10.0" +__version__ = "2.11.0" From 73913df6918684d187e4a7cb118b0284803c2551 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 19 Nov 2024 20:19:07 +0100 Subject: [PATCH 042/110] Version 2.11.1 --- CHANGES.rst | 8 +++++++- pyproject.toml | 2 +- query_exporter/__init__.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2a6bb50..d53ab22 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v2.11.1 - 2024-11-19 +==================== + +- Update ``prometheus-aioexporter`` dependency range. + + v2.11.0 - 2024-11-19 ==================== @@ -6,7 +12,7 @@ v2.11.0 - 2024-11-19 - [docker] Use Python 3.13 on Debian Bookwork as base image. - [snap] Rebase on core24. - + v2.10.0 - 2024-01-28 ==================== diff --git a/pyproject.toml b/pyproject.toml index f0b00b7..7d90fd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "argcomplete", "croniter", "jsonschema", - "prometheus-aioexporter>=2", + "prometheus-aioexporter>=2,<3", "prometheus-client", "python-dateutil", "pyyaml", diff --git a/query_exporter/__init__.py b/query_exporter/__init__.py index 59ee234..703c2ba 100644 --- a/query_exporter/__init__.py +++ b/query_exporter/__init__.py @@ -1,3 +1,3 @@ """Export Prometheus metrics generated from SQL queries.""" -__version__ = "2.11.0" +__version__ = "2.11.1" From 15632f1d068d77a3cc75eef16fee8f984cb99510 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 10 Dec 2024 09:33:47 +0100 Subject: [PATCH 043/110] chore: update ruff config --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7d90fd7..774baf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ query_exporter = [ "py.typed", "schemas/*" ] [tool.ruff] line-length = 79 -lint.select = [ "I", "RUF", "UP" ] +lint.select = [ "F", "I", "RUF", "UP" ] lint.isort.combine-as-imports = true lint.isort.force-sort-within-sections = true From 20f9df471009e2ae8216eebb93717351968fa889 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 13 Dec 2024 10:12:01 +0100 Subject: [PATCH 044/110] type fixes --- query_exporter/config.py | 4 ++-- query_exporter/loop.py | 4 ++-- query_exporter/main.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/query_exporter/config.py b/query_exporter/config.py index 9db1234..0014330 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -1,7 +1,7 @@ """Configuration management functions.""" from collections import defaultdict -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from dataclasses import dataclass from functools import reduce from importlib import resources @@ -245,7 +245,7 @@ def _get_query_metrics( ) -> list[QueryMetric]: """Return QueryMetrics for a query.""" - def _metric_labels(labels: list[str]) -> list[str]: + def _metric_labels(labels: Iterable[str]) -> list[str]: return sorted(set(labels) - extra_labels) return [ diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 75b025b..0b75078 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -255,7 +255,7 @@ def _update_metric( value = 0.0 elif isinstance(value, Decimal): value = float(value) - metric = self._config.metrics[name] + metric_config = self._config.metrics[name] all_labels = {DATABASE_LABEL: database.config.name} all_labels.update(database.config.labels) if labels: @@ -263,7 +263,7 @@ def _update_metric( labels_string = ",".join( f'{label}="{value}"' for label, value in sorted(all_labels.items()) ) - method = self._get_metric_method(metric) + method = self._get_metric_method(metric_config) self._logger.debug( f'updating metric "{name}" {method} {value} {{{labels_string}}}' ) diff --git a/query_exporter/main.py b/query_exporter/main.py index a71acd4..71f9efa 100644 --- a/query_exporter/main.py +++ b/query_exporter/main.py @@ -22,7 +22,7 @@ from .loop import QueryLoop -class QueryExporterScript(PrometheusExporterScript): # type: ignore +class QueryExporterScript(PrometheusExporterScript): """Periodically run database queries and export results to Prometheus.""" name = "query-exporter" From 99a18fd93a0c76a43bccbf1a33d54717859c3523 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 17 Dec 2024 18:13:03 +0100 Subject: [PATCH 045/110] chore: add dependabot config (#209) --- .github/dependabot.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..645c171 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From 192a97f0863967f281d3ec68737d0b711b84ea71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:15:48 +0100 Subject: [PATCH 046/110] Bump actions/checkout from 3 to 4 (#211) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46ffa78..90d9456 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Repository checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -38,7 +38,7 @@ jobs: - "3.13" steps: - name: Repository checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -63,7 +63,7 @@ jobs: - "3.13" steps: - name: Repository checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 From 287144d8b92242a1c7facbe8f91f736dfc5582bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:18:21 +0100 Subject: [PATCH 047/110] Bump actions/setup-python from 4 to 5 (#210) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90d9456..fa64357 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.13" @@ -41,7 +41,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -66,7 +66,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From 660f54b9d4e1bebe85deb640e9bec6502c83947a Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 18 Dec 2024 12:34:47 +0100 Subject: [PATCH 048/110] upate dependabot config --- .github/dependabot.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 645c171..e3a6763 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,10 +1,18 @@ version: 2 updates: - - package-ecosystem: "pip" - directory: "/" + - package-ecosystem: pip + directory: / schedule: - interval: "weekly" - - package-ecosystem: "github-actions" - directory: "/" + interval: weekly + groups: + minor: + update-types: minor + major: + update-types: major + - package-ecosystem: github-actions + directory: / schedule: - interval: "weekly" + interval: weekly + groups: + all: + update-types: major From 0b1736b4d55494506b18d2e0d51fbec152576ed2 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 20 Dec 2024 09:58:17 +0100 Subject: [PATCH 049/110] update dependabot config --- .github/dependabot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e3a6763..110271a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,13 +6,13 @@ updates: interval: weekly groups: minor: - update-types: minor + update-types: [minor] major: - update-types: major + update-types: [major] - package-ecosystem: github-actions directory: / schedule: interval: weekly groups: all: - update-types: major + update-types: [major, minor] From 14d169820c70aacdf25b92961b984d4230edd9b2 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 18 Dec 2024 12:38:34 +0100 Subject: [PATCH 050/110] define requirements.txt --- Dockerfile | 1 + pyproject.toml | 4 +++ requirements.txt | 85 +++++++++++++++++++++++++++++++++++++++++++++ snap/snapcraft.yaml | 2 ++ tox.ini | 10 ++++++ 5 files changed, 102 insertions(+) create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index 80d26b0..11d4fed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ COPY . /srcdir RUN python3 -m venv /virtualenv ENV PATH="/virtualenv/bin:$PATH" RUN pip install \ + -r /srcdir/requirements.txt \ /srcdir \ cx-Oracle \ clickhouse-sqlalchemy \ diff --git a/pyproject.toml b/pyproject.toml index 774baf3..224ef99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,3 +97,7 @@ ignore_missing_imports = true install_types = true non_interactive = true strict = true + +[tool.pip-tools] +upgrade = true +quiet = true diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a0e996a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,85 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --output-file=requirements.txt pyproject.toml +# +aiohappyeyeballs==2.4.4 + # via aiohttp +aiohttp==3.11.11 + # via + # prometheus-aioexporter + # query-exporter (pyproject.toml) +aiosignal==1.3.2 + # via aiohttp +argcomplete==3.5.2 + # via query-exporter (pyproject.toml) +attrs==24.3.0 + # via + # aiohttp + # jsonschema + # referencing +croniter==6.0.0 + # via query-exporter (pyproject.toml) +frozenlist==1.5.0 + # via + # aiohttp + # aiosignal +greenlet==3.1.1 + # via sqlalchemy +idna==3.10 + # via yarl +iniconfig==2.0.0 + # via pytest +jsonschema==4.23.0 + # via query-exporter (pyproject.toml) +jsonschema-specifications==2024.10.1 + # via jsonschema +multidict==6.1.0 + # via + # aiohttp + # yarl +packaging==24.2 + # via pytest +pluggy==1.5.0 + # via pytest +prometheus-aioexporter==2.1.0 + # via query-exporter (pyproject.toml) +prometheus-client==0.21.1 + # via + # prometheus-aioexporter + # query-exporter (pyproject.toml) +propcache==0.2.1 + # via + # aiohttp + # yarl +pytest==8.3.4 + # via toolrack +python-dateutil==2.9.0.post0 + # via + # croniter + # query-exporter (pyproject.toml) +pytz==2024.2 + # via croniter +pyyaml==6.0.2 + # via query-exporter (pyproject.toml) +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +rpds-py==0.22.3 + # via + # jsonschema + # referencing +six==1.17.0 + # via python-dateutil +sqlalchemy==2.0.36 + # via query-exporter (pyproject.toml) +toolrack==4.0.1 + # via + # prometheus-aioexporter + # query-exporter (pyproject.toml) +typing-extensions==4.12.2 + # via sqlalchemy +yarl==1.18.3 + # via aiohttp diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 57e6044..1de6703 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -61,6 +61,8 @@ parts: plugin: python source: . source-type: git + python-requirements: + - requirements.txt python-packages: - . - clickhouse-sqlalchemy diff --git a/tox.ini b/tox.ini index 7521884..6cfbdf0 100644 --- a/tox.ini +++ b/tox.ini @@ -5,12 +5,14 @@ no_package = true [testenv] deps = + -r requirements.txt .[testing] commands = pytest {posargs} [testenv:check] deps = + -r requirements.txt .[testing] mypy commands = @@ -18,6 +20,7 @@ commands = [testenv:coverage] deps = + -r requirements.txt .[testing] pytest-cov commands = @@ -45,9 +48,16 @@ commands = [testenv:run] deps = -e . + -r requirements.txt commands = {envbindir}/query-exporter {posargs} +[testenv:update-deps] +deps = + pip-tools +commands = + pip-compile --output-file requirements.txt pyproject.toml + [base] lint_files = query_exporter \ From 7dffca8c04dcce081b6024256e2256dc0c606739 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 27 Dec 2024 10:06:31 +0100 Subject: [PATCH 051/110] rename tox target to update-dependencies --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6cfbdf0..c958e83 100644 --- a/tox.ini +++ b/tox.ini @@ -52,7 +52,7 @@ deps = commands = {envbindir}/query-exporter {posargs} -[testenv:update-deps] +[testenv:update-dependencies] deps = pip-tools commands = From 56c8c63b4423a08f625468b3bcee2b94a216990b Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 16 Dec 2024 18:11:00 +0100 Subject: [PATCH 052/110] docker action --- .github/workflows/docker-image.yaml | 63 +++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/docker-image.yaml diff --git a/.github/workflows/docker-image.yaml b/.github/workflows/docker-image.yaml new file mode 100644 index 0000000..8e34db9 --- /dev/null +++ b/.github/workflows/docker-image.yaml @@ -0,0 +1,63 @@ +name: Docker image + +on: + push: + branches: + - main + - docker-action # XXX + pull_request: + branches: + - main + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=schedule + type=semver,pattern={{version}} + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=sha,prefix=sha- + + - name: Build and push Docker image + id: push + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v2 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true From 185c6ce8e5a3c45f12911ce70219f70f2e3b5121 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 27 Dec 2024 11:42:51 +0100 Subject: [PATCH 053/110] update docker workflow config --- .github/workflows/docker-image.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-image.yaml b/.github/workflows/docker-image.yaml index 8e34db9..291e9ee 100644 --- a/.github/workflows/docker-image.yaml +++ b/.github/workflows/docker-image.yaml @@ -4,7 +4,6 @@ on: push: branches: - main - - docker-action # XXX pull_request: branches: - main From 39a75af7d1c71df99fd3711d8e1842d300afff12 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 8 Nov 2024 16:18:49 +0100 Subject: [PATCH 054/110] switch to prometheus-aioexporter 3.0 (fixes #199) --- pyproject.toml | 5 +- query_exporter/__init__.py | 2 +- query_exporter/config.py | 69 +++++---- query_exporter/db.py | 112 ++++++-------- query_exporter/loop.py | 33 ++-- query_exporter/main.py | 65 ++++---- tests/config_test.py | 302 +++++++++++++----------------------- tests/conftest.py | 27 +--- tests/db_test.py | 308 ++++++++++++++++--------------------- tests/loop_test.py | 154 ++++++++----------- 10 files changed, 450 insertions(+), 627 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 224ef99..eb0cfa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,20 +41,21 @@ dynamic = [ ] dependencies = [ "aiohttp", - "argcomplete", "croniter", "jsonschema", - "prometheus-aioexporter>=2,<3", + "prometheus-aioexporter>=3", "prometheus-client", "python-dateutil", "pyyaml", "sqlalchemy>=2", + "structlog", "toolrack>=4", ] optional-dependencies.testing = [ "pytest", "pytest-asyncio", "pytest-mock", + "pytest-structlog", ] urls.changelog = "https://github.com/albertodonato/query-exporter/blob/main/CHANGES.rst" urls.homepage = "https://github.com/albertodonato/query-exporter" diff --git a/query_exporter/__init__.py b/query_exporter/__init__.py index 703c2ba..6bc2e29 100644 --- a/query_exporter/__init__.py +++ b/query_exporter/__init__.py @@ -1,3 +1,3 @@ """Export Prometheus metrics generated from SQL queries.""" -__version__ = "2.11.1" +__version__ = "3.0.0" diff --git a/query_exporter/config.py b/query_exporter/config.py index 0014330..839e19d 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -1,19 +1,15 @@ """Configuration management functions.""" from collections import defaultdict -from collections.abc import Iterable, Mapping +from collections.abc import Mapping from dataclasses import dataclass from functools import reduce from importlib import resources import itertools -from logging import Logger import os from pathlib import Path import re -from typing import ( - IO, - Any, -) +import typing as t from urllib.parse import ( quote_plus, urlencode, @@ -21,6 +17,7 @@ import jsonschema from prometheus_aioexporter import MetricConfig +import structlog import yaml from .db import ( @@ -94,14 +91,20 @@ class Config: Environ = Mapping[str, str] # Content for the "parameters" config option -ParametersConfig = list[dict[str, Any]] | dict[str, list[dict[str, Any]]] +ParametersConfig = list[dict[str, t.Any]] | dict[str, list[dict[str, t.Any]]] def load_config( - config_fd: IO[str], logger: Logger, env: Environ = os.environ + config_path: Path, + logger: structlog.stdlib.BoundLogger | None = None, + env: Environ = os.environ, ) -> Config: """Load YAML config from file.""" - data = defaultdict(dict, yaml.safe_load(config_fd)) + if logger is None: + logger = structlog.get_logger() + + with config_path.open() as fd: + data = defaultdict(dict, yaml.safe_load(fd)) _validate_config(data) databases, database_labels = _get_databases(data["databases"], env) extra_labels = frozenset([DATABASE_LABEL]) | database_labels @@ -115,7 +118,7 @@ def load_config( def _get_databases( - configs: dict[str, dict[str, Any]], env: Environ + configs: dict[str, dict[str, t.Any]], env: Environ ) -> tuple[dict[str, DataBaseConfig], frozenset[str]]: """Return a dict mapping names to database configs, and a set of database labels.""" databases = {} @@ -147,7 +150,7 @@ def _get_databases( def _get_metrics( - metrics: dict[str, dict[str, Any]], extra_labels: frozenset[str] + metrics: dict[str, dict[str, t.Any]], extra_labels: frozenset[str] ) -> dict[str, MetricConfig]: """Return a dict mapping metric names to their configuration.""" configs = {} @@ -179,7 +182,7 @@ def _get_metrics( def _validate_metric_config( - name: str, config: dict[str, Any], extra_labels: frozenset[str] + name: str, config: dict[str, t.Any], extra_labels: frozenset[str] ) -> None: """Validate a metric configuration stanza.""" if name in GLOBAL_METRICS: @@ -194,7 +197,7 @@ def _validate_metric_config( def _get_queries( - configs: dict[str, dict[str, Any]], + configs: dict[str, dict[str, t.Any]], database_names: frozenset[str], metrics: dict[str, MetricConfig], extra_labels: frozenset[str], @@ -239,13 +242,13 @@ def _get_queries( def _get_query_metrics( - config: dict[str, Any], + config: dict[str, t.Any], metrics: dict[str, MetricConfig], extra_labels: frozenset[str], ) -> list[QueryMetric]: """Return QueryMetrics for a query.""" - def _metric_labels(labels: Iterable[str]) -> list[str]: + def _metric_labels(labels: t.Iterable[str]) -> list[str]: return sorted(set(labels) - extra_labels) return [ @@ -256,7 +259,7 @@ def _metric_labels(labels: Iterable[str]) -> list[str]: def _validate_query_config( name: str, - config: dict[str, Any], + config: dict[str, t.Any], database_names: frozenset[str], metric_names: frozenset[str], ) -> None: @@ -313,7 +316,7 @@ def _convert_interval(interval: int | str | None) -> int | None: return int(interval) * multiplier -def _resolve_dsn(dsn: str | dict[str, Any], env: Environ) -> str: +def _resolve_dsn(dsn: str | dict[str, t.Any], env: Environ) -> str: """Build and resolve the database DSN string from the right source.""" def from_env(varname: str) -> str: @@ -347,7 +350,7 @@ def from_file(filename: str) -> str: return dsn -def _build_dsn(details: dict[str, Any]) -> str: +def _build_dsn(details: dict[str, t.Any]) -> str: """Build a DSN string from details.""" url = f"{details['dialect']}://" user = details.get("user") @@ -375,7 +378,7 @@ def _build_dsn(details: dict[str, Any]) -> str: return url -def _validate_config(config: dict[str, Any]) -> None: +def _validate_config(config: dict[str, t.Any]) -> None: schema_file = resources.files("query_exporter") / "schemas" / "config.yaml" schema = yaml.safe_load(schema_file.read_bytes()) try: @@ -385,7 +388,9 @@ def _validate_config(config: dict[str, Any]) -> None: raise ConfigError(f"Invalid config at {path}: {e.message}") -def _warn_if_unused(config: Config, logger: Logger) -> None: +def _warn_if_unused( + config: Config, logger: structlog.stdlib.BoundLogger +) -> None: """Warn if there are unused databases or metrics defined.""" used_dbs: set[str] = set() used_metrics: set[str] = set() @@ -393,21 +398,25 @@ def _warn_if_unused(config: Config, logger: Logger) -> None: used_dbs.update(query.databases) used_metrics.update(metric.name for metric in query.metrics) - unused_dbs = sorted(set(config.databases) - used_dbs) - if unused_dbs: + if unused_dbs := sorted(set(config.databases) - used_dbs): logger.warning( - f"unused entries in \"databases\" section: {', '.join(unused_dbs)}" + "unused config entries", + section="databases", + entries=unused_dbs, ) - unused_metrics = sorted( + if unused_metrics := sorted( set(config.metrics) - GLOBAL_METRICS - used_metrics - ) - if unused_metrics: + ): logger.warning( - f"unused entries in \"metrics\" section: {', '.join(unused_metrics)}" + "unused config entries", + section="metrics", + entries=unused_metrics, ) -def _get_parameters_sets(parameters: ParametersConfig) -> list[dict[str, Any]]: +def _get_parameters_sets( + parameters: ParametersConfig, +) -> list[dict[str, t.Any]]: """Return an sequence of set of paramters with their values.""" if isinstance(parameters, dict): return _get_parameters_matrix(parameters) @@ -415,8 +424,8 @@ def _get_parameters_sets(parameters: ParametersConfig) -> list[dict[str, Any]]: def _get_parameters_matrix( - parameters: dict[str, list[dict[str, Any]]], -) -> list[dict[str, Any]]: + parameters: dict[str, list[dict[str, t.Any]]], +) -> list[dict[str, t.Any]]: """Return parameters combinations from a matrix.""" # first, flatten dict like # diff --git a/query_exporter/db.py b/query_exporter/db.py index a738ace..5b4b90f 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -12,8 +12,6 @@ ) from functools import partial from itertools import chain -import logging -import sys from threading import ( Thread, current_thread, @@ -22,14 +20,8 @@ perf_counter, time, ) -from traceback import format_tb from types import TracebackType -from typing import ( - Any, - NamedTuple, - Self, - cast, -) +import typing as t from croniter import croniter from sqlalchemy import ( @@ -48,6 +40,7 @@ NoSuchModuleError, ) from sqlalchemy.sql.elements import TextClause +import structlog #: Timeout for a query QueryTimeout = int | float @@ -79,11 +72,6 @@ class DataBaseQueryError(DataBaseError): class QueryTimeoutExpired(Exception): """Query execution timeout expired.""" - def __init__(self, query_name: str, timeout: QueryTimeout) -> None: - super().__init__( - f'Execution for query "{query_name}" expired after {timeout} seconds' - ) - class InvalidResultCount(Exception): """Number of results from a query don't match metrics count.""" @@ -150,7 +138,7 @@ def __post_init__(self) -> None: create_db_engine(self.dsn) -def create_db_engine(dsn: str, **kwargs: Any) -> Engine: +def create_db_engine(dsn: str, **kwargs: t.Any) -> Engine: """Create the database engine, validating the DSN""" try: return create_engine(dsn, **kwargs) @@ -160,42 +148,42 @@ def create_db_engine(dsn: str, **kwargs: Any) -> Engine: raise DataBaseError(f'Invalid database DSN: "{dsn}"') -class QueryMetric(NamedTuple): +class QueryMetric(t.NamedTuple): """Metric details for a Query.""" name: str labels: Iterable[str] -class QueryResults(NamedTuple): +class QueryResults(t.NamedTuple): """Results of a database query.""" keys: list[str] - rows: Sequence[Row[Any]] + rows: Sequence[Row[t.Any]] timestamp: float | None = None latency: float | None = None @classmethod - def from_result(cls, result: CursorResult[Any]) -> Self: + def from_result(cls, result: CursorResult[t.Any]) -> t.Self: """Return a QueryResults from results for a query.""" timestamp = time() keys: list[str] = [] - rows: Sequence[Row[Any]] = [] + rows: Sequence[Row[t.Any]] = [] if result.returns_rows: keys, rows = list(result.keys()), result.all() latency = result.connection.info.get("query_latency", None) return cls(keys, rows, timestamp=timestamp, latency=latency) -class MetricResult(NamedTuple): +class MetricResult(t.NamedTuple): """A result for a metric from a query.""" metric: str - value: Any + value: t.Any labels: dict[str, str] -class MetricResults(NamedTuple): +class MetricResults(t.NamedTuple): """Collection of metric results for a query.""" results: list[MetricResult] @@ -212,7 +200,7 @@ def __init__( databases: list[str], metrics: list[QueryMetric], sql: str, - parameters: dict[str, Any] | None = None, + parameters: dict[str, t.Any] | None = None, timeout: QueryTimeout | None = None, interval: int | None = None, schedule: str | None = None, @@ -288,7 +276,7 @@ class WorkerAction: """An action to be called in the worker thread.""" def __init__( - self, func: Callable[..., Any], *args: Any, **kwargs: Any + self, func: Callable[..., t.Any], *args: t.Any, **kwargs: t.Any ) -> None: self._func = partial(func, *args, **kwargs) self._loop = asyncio.get_event_loop() @@ -297,6 +285,8 @@ def __init__( def __str__(self) -> str: return self._func.func.__name__ + __repr__ = __str__ + def __call__(self) -> None: """Call the action asynchronously in a thread-safe way.""" try: @@ -306,11 +296,13 @@ def __call__(self) -> None: else: self._call_threadsafe(self._future.set_result, result) - async def result(self) -> Any: + async def result(self) -> t.Any: """Wait for completion and return the action result.""" return await self._future - def _call_threadsafe(self, call: Callable[..., Any], *args: Any) -> None: + def _call_threadsafe( + self, call: Callable[..., t.Any], *args: t.Any + ) -> None: self._loop.call_soon_threadsafe(partial(call, *args)) @@ -324,11 +316,11 @@ def __init__( self, dbname: str, engine: Engine, - logger: logging.Logger = logging.getLogger(), + logger: structlog.stdlib.BoundLogger | None = None, ) -> None: self.dbname = dbname self.engine = engine - self.logger = logger + self.logger = logger or structlog.get_logger() self._loop = asyncio.get_event_loop() self._queue: asyncio.Queue[WorkerAction] = asyncio.Queue() @@ -356,7 +348,7 @@ async def close(self) -> None: async def execute( self, sql: TextClause, - parameters: dict[str, Any] | None = None, + parameters: dict[str, t.Any] | None = None, ) -> QueryResults: """Execute a query, returning results.""" if parameters is None: @@ -383,8 +375,8 @@ def _connect(self) -> None: self._conn = self.engine.connect() def _execute( - self, sql: TextClause, parameters: dict[str, Any] - ) -> CursorResult[Any]: + self, sql: TextClause, parameters: dict[str, t.Any] + ) -> CursorResult[t.Any]: assert self._conn return self._conn.execute(sql, parameters) @@ -396,27 +388,24 @@ def _close(self) -> None: def _run(self) -> None: """The worker thread function.""" - - def debug(message: str) -> None: - self.logger.debug(f'worker "{current_thread().name}": {message}') - - debug(f"started with ID {current_thread().native_id}") + logger = self.logger.bind(worker_id=current_thread().native_id) + logger.debug("start") while True: future = asyncio.run_coroutine_threadsafe( self._queue.get(), self._loop ) action = future.result() - debug(f'received action "{action}"') + logger.debug("action received", action=str(action)) action() self._loop.call_soon_threadsafe(self._queue.task_done) if self._conn is None: # the connection has been closed, exit the thread - debug("shutting down") + logger.debug("shutdown") return async def _call_in_thread( - self, func: Callable[..., Any], *args: Any, **kwargs: Any - ) -> Any: + self, func: Callable[..., t.Any], *args: t.Any, **kwargs: t.Any + ) -> t.Any: """Call a sync action in the worker thread.""" call = WorkerAction(func, *args, **kwargs) await self._queue.put(call) @@ -432,10 +421,12 @@ class DataBase: def __init__( self, config: DataBaseConfig, - logger: logging.Logger = logging.getLogger(), + logger: structlog.stdlib.BoundLogger | None = None, ) -> None: self.config = config - self.logger = logger + if logger is None: + logger = structlog.get_logger() + self.logger = logger.bind(database=self.config.name) self._connect_lock = asyncio.Lock() execution_options = {} if self.config.autocommit: @@ -447,7 +438,7 @@ def __init__( self._conn = DataBaseConnection(self.config.name, engine, self.logger) self._setup_query_latency_tracking(engine) - async def __aenter__(self) -> Self: + async def __aenter__(self) -> t.Self: await self.connect() return self @@ -475,7 +466,7 @@ async def connect(self) -> None: except Exception as error: raise self._db_error(error, exc_class=DataBaseConnectError) - self.logger.debug(f'connected to database "{self.config.name}"') + self.logger.debug("connected") for sql in self.config.connect_sql: try: await self.execute_sql(sql) @@ -496,9 +487,7 @@ async def close(self) -> None: async def execute(self, query: Query) -> MetricResults: """Execute a query.""" await self.connect() - self.logger.debug( - f'running query "{query.name}" on database "{self.config.name}"' - ) + self.logger.debug("run query", query=query.name) self._pending_queries += 1 try: query_results = await self.execute_sql( @@ -506,9 +495,8 @@ async def execute(self, query: Query) -> MetricResults: ) return query.results(query_results) except TimeoutError: - raise self._query_timeout_error( - query.name, cast(QueryTimeout, query.timeout) - ) + self.logger.warning("query timeout", query=query.name) + raise QueryTimeoutExpired() except Exception as error: raise self._query_db_error( query.name, error, fatal=isinstance(error, FATAL_ERRORS) @@ -522,7 +510,7 @@ async def execute(self, query: Query) -> MetricResults: async def execute_sql( self, sql: str, - parameters: dict[str, Any] | None = None, + parameters: dict[str, t.Any] | None = None, timeout: QueryTimeout | None = None, ) -> QueryResults: """Execute a raw SQL query.""" @@ -534,7 +522,7 @@ async def execute_sql( async def _close(self) -> None: # ensure the connection with the DB is actually closed await self._conn.close() - self.logger.debug(f'disconnected from database "{self.config.name}"') + self.logger.debug("disconnected") def _setup_query_latency_tracking(self, engine: Engine) -> None: @event.listens_for(engine, "before_cursor_execute") # type: ignore @@ -559,21 +547,11 @@ def _query_db_error( ) -> DataBaseError: """Create and log a DataBaseError for a failed query.""" message = self._error_message(error) - self.logger.error( - f'query "{query_name}" on database "{self.config.name}" failed: ' - + message + self.logger.exception( + "query failed", query=query_name, error=message, exception=error ) - _, _, traceback = sys.exc_info() - self.logger.debug("".join(format_tb(traceback))) return DataBaseQueryError(message, fatal=fatal) - def _query_timeout_error( - self, query_name: str, timeout: QueryTimeout - ) -> QueryTimeoutExpired: - error = QueryTimeoutExpired(query_name, timeout) - self.logger.warning(str(error)) - raise error - def _db_error( self, error: str | Exception, @@ -582,9 +560,7 @@ def _db_error( ) -> DataBaseError: """Create and log a DataBaseError.""" message = self._error_message(error) - self.logger.error( - f'error from database "{self.config.name}": {message}' - ) + self.logger.exception("database error", exception=error) return exc_class(message, fatal=fatal) def _error_message(self, error: str | Exception) -> str: diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 0b75078..bc8c582 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -5,12 +5,8 @@ from collections.abc import Iterator, Mapping from datetime import datetime from decimal import Decimal -from logging import Logger import time -from typing import ( - Any, - cast, -) +import typing as t from croniter import croniter from dateutil.tz import gettz @@ -20,6 +16,7 @@ ) from prometheus_client import Counter from prometheus_client.metrics import MetricWrapperBase +import structlog from toolrack.aio import ( PeriodicCall, TimedCall, @@ -82,7 +79,7 @@ def expire_series( """ expired = {} for name, metric_last_seen in self._last_seen.items(): - expiration = cast(int, self._expirations[name]) + expiration = t.cast(int, self._expirations[name]) expired_labels = [ label_values for label_values, last_seen in metric_last_seen.items() @@ -107,11 +104,11 @@ def __init__( self, config: Config, registry: MetricsRegistry, - logger: Logger, + logger: structlog.stdlib.BoundLogger | None = None, ): self._config = config self._registry = registry - self._logger = logger + self._logger = logger or structlog.get_logger() self._timed_queries: list[Query] = [] self._aperiodic_queries: list[Query] = [] # map query names to their TimedCalls @@ -203,8 +200,9 @@ async def _execute_query(self, query: Query, dbname: str) -> None: self._increment_queries_count(db, query, "error") if error.fatal: self._logger.debug( - f'removing failed query "{query.name}" ' - f'for database "{dbname}"' + "removing failed query", + query=query.name, + database=dbname, ) self._doomed_queries[query.name].add(dbname) else: @@ -246,7 +244,7 @@ def _update_metric( self, database: DataBase, name: str, - value: Any, + value: t.Any, labels: Mapping[str, str] | None = None, ) -> None: """Update value for a metric.""" @@ -260,12 +258,13 @@ def _update_metric( all_labels.update(database.config.labels) if labels: all_labels.update(labels) - labels_string = ",".join( - f'{label}="{value}"' for label, value in sorted(all_labels.items()) - ) method = self._get_metric_method(metric_config) self._logger.debug( - f'updating metric "{name}" {method} {value} {{{labels_string}}}' + "updating metric", + metric=name, + method=method, + value=value, + labels=all_labels, ) metric = self._registry.get_metric(name, labels=all_labels) self._update_metric_value(metric, method, value) @@ -287,11 +286,11 @@ def _get_metric_method(self, metric: MetricConfig) -> str: return method def _update_metric_value( - self, metric: MetricWrapperBase, method: str, value: Any + self, metric: MetricWrapperBase, method: str, value: t.Any ) -> None: if metric._type == "counter" and method == "set": # counters can only be incremented, directly set the underlying value - cast(Counter, metric)._value.set(value) + t.cast(Counter, metric)._value.set(value) else: getattr(metric, method)(value) diff --git a/query_exporter/main.py b/query_exporter/main.py index 71f9efa..c129f69 100644 --- a/query_exporter/main.py +++ b/query_exporter/main.py @@ -1,17 +1,16 @@ """Script entry point.""" -import argparse from functools import partial -from typing import IO +from pathlib import Path from aiohttp.web import Application -from argcomplete import autocomplete +import click from prometheus_aioexporter import ( + Arguments, InvalidMetricType, MetricConfig, PrometheusExporterScript, ) -from toolrack.script import ErrorExitMessage from . import __version__ from .config import ( @@ -26,36 +25,42 @@ class QueryExporterScript(PrometheusExporterScript): """Periodically run database queries and export results to Prometheus.""" name = "query-exporter" + version = __version__ description = __doc__ default_port = 9560 + envvar_prefix = "QE" - def configure_argument_parser( - self, parser: argparse.ArgumentParser - ) -> None: - parser.add_argument( - "config", type=argparse.FileType("r"), help="configuration file" - ) - parser.add_argument( - "-V", - "--version", - action="version", - version=f"%(prog)s {__version__}", - ) - parser.add_argument( - "--check-only", - action="store_true", - help="only check configuration, don't run the exporter", - ) - autocomplete(parser) + def command_line_parameters(self) -> list[click.Parameter]: + return [ + click.Option( + ["--check-only"], + type=bool, + help="only check configuration, don't run the exporter", + is_flag=True, + show_default=True, + show_envvar=True, + ), + click.Option( + ["--config"], + type=click.Path( + exists=True, + dir_okay=False, + path_type=Path, + ), + help="configuration file", + default=Path("config.yaml"), + show_default=True, + show_envvar=True, + ), + ] - def configure(self, args: argparse.Namespace) -> None: + def configure(self, args: Arguments) -> None: self.config = self._load_config(args.config) if args.check_only: - self.exit() + raise SystemExit(0) self.create_metrics(self.config.metrics.values()) async def on_application_startup(self, application: Application) -> None: - self.logger.info(f"version {__version__} starting up") query_loop = QueryLoop(self.config, self.registry, self.logger) application["exporter"].set_metric_update_handler( partial(self._update_handler, query_loop) @@ -73,15 +78,13 @@ async def _update_handler( await query_loop.run_aperiodic_queries() query_loop.clear_expired_series() - def _load_config(self, config_file: IO[str]) -> Config: + def _load_config(self, config_file: Path) -> Config: """Load the application configuration.""" try: - config = load_config(config_file, self.logger) + return load_config(config_file, self.logger) except (InvalidMetricType, ConfigError) as error: - raise ErrorExitMessage(str(error)) - finally: - config_file.close() - return config + self.logger.error("invalid config", error=str(error)) + raise SystemExit(1) script = QueryExporterScript() diff --git a/tests/config_test.py b/tests/config_test.py index c4333d5..2e97570 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -1,10 +1,9 @@ from collections.abc import Callable, Iterator -import logging from pathlib import Path -from typing import Any +import typing as t import pytest -from pytest import LogCaptureFixture +from pytest_structlog import StructuredLogCapture import yaml from query_exporter.config import ( @@ -20,14 +19,7 @@ @pytest.fixture -def logger(caplog: LogCaptureFixture) -> Iterator[logging.Logger]: - with caplog.at_level("DEBUG"): - yield logging.getLogger() - caplog.clear() - - -@pytest.fixture -def config_full() -> Iterator[dict[str, Any]]: +def config_full() -> Iterator[dict[str, t.Any]]: yield { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "gauge", "labels": ["l1", "l2"]}}, @@ -42,14 +34,14 @@ def config_full() -> Iterator[dict[str, Any]]: } -ConfigWriter = Callable[[dict[str, Any]], Path] +ConfigWriter = Callable[[dict[str, t.Any]], Path] @pytest.fixture def write_config(tmp_path: Path) -> Iterator[ConfigWriter]: path = tmp_path / "config" - def write(data: dict[str, Any]) -> Path: + def write(data: dict[str, t.Any]) -> Path: path.write_text(yaml.dump(data), "utf-8") return path @@ -136,10 +128,7 @@ def write(data: dict[str, Any]) -> Path: class TestLoadConfig: - def test_load_databases_section( - self, logger: logging.Logger, write_config: ConfigWriter - ) -> None: - """The 'databases' section is loaded from the config file.""" + def test_load_databases_section(self, write_config: ConfigWriter) -> None: cfg = { "databases": { "db1": {"dsn": "sqlite:///foo"}, @@ -153,8 +142,7 @@ def test_load_databases_section( "queries": {}, } config_file = write_config(cfg) - with config_file.open() as fd: - config = load_config(fd, logger) + config = load_config(config_file) assert {"db1", "db2"} == set(config.databases) database1 = config.databases["db1"] database2 = config.databases["db2"] @@ -168,54 +156,49 @@ def test_load_databases_section( assert not database2.autocommit def test_load_databases_dsn_from_env( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """The database DSN can be loaded from env.""" cfg = { "databases": {"db1": {"dsn": "env:FOO"}}, "metrics": {}, "queries": {}, } config_file = write_config(cfg) - with config_file.open() as fd: - config = load_config(fd, logger, env={"FOO": "sqlite://"}) + config = load_config(config_file, env={"FOO": "sqlite://"}) assert config.databases["db1"].dsn == "sqlite://" def test_load_databases_missing_dsn( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if the 'dsn' key is missing for a database.""" - cfg: dict[str, Any] = { + cfg: dict[str, t.Any] = { "databases": {"db1": {}}, "metrics": {}, "queries": {}, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == "Invalid config at databases/db1: 'dsn' is a required property" ) def test_load_databases_invalid_dsn( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if the DSN is invalid.""" cfg = { "databases": {"db1": {"dsn": "invalid"}}, "metrics": {}, "queries": {}, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert str(err.value) == 'Invalid database DSN: "invalid"' def test_load_databases_dsn_details( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """The DSN can be specified as a dictionary.""" cfg = { "databases": { "db1": { @@ -229,14 +212,12 @@ def test_load_databases_dsn_details( "queries": {}, } config_file = write_config(cfg) - with config_file.open() as fd: - config = load_config(fd, logger) + config = load_config(config_file) assert config.databases["db1"].dsn == "sqlite:///path/to/file" def test_load_databases_dsn_details_only_dialect( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """Only the "dialect" key is required in DSN.""" cfg = { "databases": { "db1": { @@ -249,45 +230,40 @@ def test_load_databases_dsn_details_only_dialect( "queries": {}, } config_file = write_config(cfg) - with config_file.open() as fd: - config = load_config(fd, logger) + config = load_config(config_file) assert config.databases["db1"].dsn == "sqlite://" def test_load_databases_dsn_invalid_env( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if the DSN from environment is invalid.""" cfg = { "databases": {"db1": {"dsn": "env:NOT-VALID"}}, "metrics": {}, "queries": {}, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert str(err.value) == 'Invalid variable name: "NOT-VALID"' def test_load_databases_dsn_undefined_env( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if the environ variable for DSN is undefined.""" cfg = { "databases": {"db1": {"dsn": "env:FOO"}}, "metrics": {}, "queries": {}, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger, env={}) + with pytest.raises(ConfigError) as err: + load_config(config_file, env={}) assert str(err.value) == 'Undefined variable: "FOO"' def test_load_databases_dsn_from_file( self, tmp_path: Path, - logger: logging.Logger, write_config: ConfigWriter, ) -> None: - """The database DSN can be loaded from a file.""" dsn = "sqlite:///foo" dsn_path = tmp_path / "dsn" dsn_path.write_text(dsn) @@ -297,31 +273,28 @@ def test_load_databases_dsn_from_file( "queries": {}, } config_file = write_config(cfg) - with config_file.open() as fd: - config = load_config(fd, logger) + config = load_config(config_file) assert config.databases["db1"].dsn == dsn def test_load_databases_dsn_from_file_not_found( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if the DSN file can't be read.""" cfg = { "databases": {"db1": {"dsn": "file:/not/found"}}, "metrics": {}, "queries": {}, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == 'Unable to read dsn file : "/not/found": No such file or directory' ) def test_load_databases_no_labels( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """If no labels are defined, an empty dict is returned.""" cfg = { "databases": { "db": { @@ -332,15 +305,11 @@ def test_load_databases_no_labels( "queries": {}, } config_file = write_config(cfg) - with config_file.open() as fd: - result = load_config(fd, logger) + result = load_config(config_file) db = result.databases["db"] assert db.labels == {} - def test_load_databases_labels( - self, logger: logging.Logger, write_config: ConfigWriter - ) -> None: - """Labels can be defined for databases.""" + def test_load_databases_labels(self, write_config: ConfigWriter) -> None: cfg = { "databases": { "db": { @@ -352,15 +321,13 @@ def test_load_databases_labels( "queries": {}, } config_file = write_config(cfg) - with config_file.open() as fd: - result = load_config(fd, logger) + result = load_config(config_file) db = result.databases["db"] assert db.labels == {"label1": "value1", "label2": "value2"} def test_load_databases_labels_not_all_same( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """If not all databases have the same labels, an error is raised.""" cfg = { "databases": { "db1": { @@ -376,14 +343,13 @@ def test_load_databases_labels_not_all_same( "queries": {}, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger, env={}) + with pytest.raises(ConfigError) as err: + load_config(config_file, env={}) assert str(err.value) == "Not all databases define the same labels" def test_load_databases_connect_sql( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """Databases can have queries defined to run on connection.""" cfg = { "databases": { "db": { @@ -395,14 +361,10 @@ def test_load_databases_connect_sql( "queries": {}, } config_file = write_config(cfg) - with config_file.open() as fd: - result = load_config(fd, logger, env={}) + result = load_config(config_file, env={}) assert result.databases["db"].connect_sql == ["SELECT 1", "SELECT 2"] - def test_load_metrics_section( - self, logger: logging.Logger, write_config: ConfigWriter - ) -> None: - """The 'metrics' section is loaded from the config file.""" + def test_load_metrics_section(self, write_config: ConfigWriter) -> None: cfg = { "databases": {"db1": {"dsn": "sqlite://"}}, "metrics": { @@ -427,8 +389,7 @@ def test_load_metrics_section( "queries": {}, } config_file = write_config(cfg) - with config_file.open() as fd: - result = load_config(fd, logger) + result = load_config(config_file) metric1 = result.metrics["metric1"] assert metric1.type == "summary" assert metric1.description == "metric one" @@ -455,34 +416,32 @@ def test_load_metrics_section( assert result.metrics.get(QUERIES_METRIC_NAME) is not None def test_load_metrics_overlap_reserved_label( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if reserved labels are used.""" cfg = { "databases": {"db1": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "gauge", "labels": ["database"]}}, "queries": {}, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == 'Labels for metric "m" overlap with reserved/database ones: database' ) def test_load_metrics_overlap_database_label( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if database labels are used for metrics.""" cfg = { "databases": {"db1": {"dsn": "sqlite://", "labels": {"l1": "v1"}}}, "metrics": {"m": {"type": "gauge", "labels": ["l1"]}}, "queries": {}, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == 'Labels for metric "m" overlap with reserved/database ones: l1' @@ -491,25 +450,22 @@ def test_load_metrics_overlap_database_label( @pytest.mark.parametrize("global_name", list(GLOBAL_METRICS)) def test_load_metrics_reserved_name( self, - config_full: dict[str, Any], - logger: logging.Logger, + config_full: dict[str, t.Any], write_config: ConfigWriter, global_name: str, ) -> None: - """An error is raised if a reserved label name is used.""" config_full["metrics"][global_name] = {"type": "counter"} config_file = write_config(config_full) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == f'Label name "{global_name} is reserved for builtin metric' ) def test_load_metrics_unsupported_type( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if an unsupported metric type is passed.""" cfg = { "databases": {"db1": {"dsn": "sqlite://"}}, "metrics": { @@ -518,17 +474,14 @@ def test_load_metrics_unsupported_type( "queries": {}, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert str(err.value) == ( "Invalid config at metrics/metric1/type: 'info' is not one of " "['counter', 'enum', 'gauge', 'histogram', 'summary']" ) - def test_load_queries_section( - self, logger: logging.Logger, write_config: ConfigWriter - ) -> None: - """The 'queries' section is loaded from the config file.""" + def test_load_queries_section(self, write_config: ConfigWriter) -> None: cfg = { "databases": { "db1": {"dsn": "sqlite:///foo"}, @@ -554,8 +507,7 @@ def test_load_queries_section( }, } config_file = write_config(cfg) - with config_file.open() as fd: - result = load_config(fd, logger) + result = load_config(config_file) query1 = result.queries["q1"] assert query1.name == "q1" assert query1.databases == ["db1"] @@ -570,9 +522,8 @@ def test_load_queries_section( assert query2.parameters == {} def test_load_queries_section_with_parameters( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """Queries can have parameters.""" cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary", "labels": ["l"]}}, @@ -590,8 +541,7 @@ def test_load_queries_section_with_parameters( }, } config_file = write_config(cfg) - with config_file.open() as fd: - result = load_config(fd, logger) + result = load_config(config_file) query1 = result.queries["q[params0]"] assert query1.name == "q[params0]" assert query1.databases == ["db"] @@ -612,9 +562,8 @@ def test_load_queries_section_with_parameters( } def test_load_queries_section_with_parameters_matrix( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """Queries can have parameters matrix.""" cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary", "labels": ["l"]}}, @@ -632,8 +581,7 @@ def test_load_queries_section_with_parameters_matrix( }, } config_file = write_config(cfg) - with config_file.open() as fd: - result = load_config(fd, logger) + result = load_config(config_file) assert len(result.queries) == 4 @@ -676,9 +624,8 @@ def test_load_queries_section_with_parameters_matrix( } def test_load_queries_section_with_wrong_parameters( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if query parameters don't match.""" cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary", "labels": ["l"]}}, @@ -696,17 +643,16 @@ def test_load_queries_section_with_wrong_parameters( }, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == 'Parameters for query "q[params0]" don\'t match those from SQL' ) def test_load_queries_section_with_schedule_and_interval( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if query schedule and interval are both present.""" cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary"}}, @@ -721,17 +667,16 @@ def test_load_queries_section_with_schedule_and_interval( }, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == 'Invalid schedule for query "q": both interval and schedule specified' ) def test_load_queries_section_invalid_schedule( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """An error is raised if query schedule has wrong format.""" cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary"}}, @@ -745,8 +690,8 @@ def test_load_queries_section_invalid_schedule( }, } config_file = write_config(cfg) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == 'Invalid schedule for query "q": invalid schedule format' @@ -754,15 +699,12 @@ def test_load_queries_section_invalid_schedule( def test_load_queries_section_timeout( self, - logger: logging.Logger, - config_full: dict[str, Any], + config_full: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - """Query configuration can include a timeout.""" config_full["queries"]["q"]["timeout"] = 2.0 config_file = write_config(config_full) - with config_file.open() as fd: - result = load_config(fd, logger) + result = load_config(config_file) query1 = result.queries["q"] assert query1.timeout == 2.0 @@ -785,17 +727,15 @@ def test_load_queries_section_timeout( ) def test_load_queries_section_invalid_timeout( self, - logger: logging.Logger, - config_full: dict[str, Any], + config_full: dict[str, t.Any], write_config: ConfigWriter, timeout: float, error_message: str, ) -> None: - """An error is raised if query timeout is invalid.""" config_full["queries"]["q"]["timeout"] = timeout config_file = write_config(config_full) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert str(err.value) == error_message @pytest.mark.parametrize( @@ -835,41 +775,39 @@ def test_load_queries_section_invalid_timeout( ) def test_configuration_incorrect( self, - logger: logging.Logger, write_config: ConfigWriter, - config: dict[str, Any], + config: dict[str, t.Any], error_message: str, ) -> None: - """An error is raised if configuration is incorrect.""" config_file = write_config(config) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert str(err.value) == error_message def test_configuration_warning_unused( self, - caplog: LogCaptureFixture, - logger: logging.Logger, - config_full: dict[str, Any], + log: StructuredLogCapture, + config_full: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - """A warning is logged if unused entries are present in config.""" config_full["databases"]["db2"] = {"dsn": "sqlite://"} config_full["databases"]["db3"] = {"dsn": "sqlite://"} config_full["metrics"]["m2"] = {"type": "gauge"} config_full["metrics"]["m3"] = {"type": "gauge"} config_file = write_config(config_full) - with config_file.open() as fd: - load_config(fd, logger) - assert caplog.messages == [ - 'unused entries in "databases" section: db2, db3', - 'unused entries in "metrics" section: m2, m3', - ] + load_config(config_file) + assert log.has( + "unused config entries", + section="databases", + entries=["db2", "db3"], + ) + assert log.has( + "unused config entries", section="metrics", entries=["m2", "m3"] + ) def test_load_queries_missing_interval_default_to_none( - self, logger: logging.Logger, write_config: ConfigWriter + self, write_config: ConfigWriter ) -> None: - """If the interval is not specified, it defaults to None.""" cfg = { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "summary"}}, @@ -878,8 +816,7 @@ def test_load_queries_missing_interval_default_to_none( }, } config_file = write_config(cfg) - with config_file.open() as fd: - config = load_config(fd, logger) + config = load_config(config_file) assert config.queries["q"].interval is None @pytest.mark.parametrize( @@ -896,45 +833,37 @@ def test_load_queries_missing_interval_default_to_none( ) def test_load_queries_interval( self, - logger: logging.Logger, - config_full: dict[str, Any], + config_full: dict[str, t.Any], write_config: ConfigWriter, interval: str | int | None, value: int | None, ) -> None: - """The query interval can be specified with suffixes.""" config_full["queries"]["q"]["interval"] = interval config_file = write_config(config_full) - with config_file.open() as fd: - config = load_config(fd, logger) + config = load_config(config_file) assert config.queries["q"].interval == value def test_load_queries_interval_not_specified( self, - logger: logging.Logger, - config_full: dict[str, Any], + config_full: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - """If the interval is not specified, it's set to None.""" del config_full["queries"]["q"]["interval"] config_file = write_config(config_full) - with config_file.open() as fd: - config = load_config(fd, logger) + config = load_config(config_file) assert config.queries["q"].interval is None @pytest.mark.parametrize("interval", ["1x", "wrong", "1.5m"]) def test_load_queries_invalid_interval_string( self, - logger: logging.Logger, - config_full: dict[str, Any], + config_full: dict[str, t.Any], write_config: ConfigWriter, interval: str, ) -> None: - """An invalid string query interval raises an error.""" config_full["queries"]["q"]["interval"] = interval config_file = write_config(config_full) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert str(err.value) == ( "Invalid config at queries/q/interval: " f"'{interval}' does not match '^[0-9]+[smhd]?$'" @@ -943,16 +872,14 @@ def test_load_queries_invalid_interval_string( @pytest.mark.parametrize("interval", [0, -20]) def test_load_queries_invalid_interval_number( self, - logger: logging.Logger, - config_full: dict[str, Any], + config_full: dict[str, t.Any], write_config: ConfigWriter, interval: int, ) -> None: - """An invalid integer query interval raises an error.""" config_full["queries"]["q"]["interval"] = interval config_file = write_config(config_full) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == f"Invalid config at queries/q/interval: {interval} is less than the minimum of 1" @@ -960,15 +887,13 @@ def test_load_queries_invalid_interval_number( def test_load_queries_no_metrics( self, - logger: logging.Logger, - config_full: dict[str, Any], + config_full: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - """An error is raised if no metrics are configured.""" config_full["queries"]["q"]["metrics"] = [] config_file = write_config(config_full) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == "Invalid config at queries/q/metrics: [] should be non-empty" @@ -976,15 +901,13 @@ def test_load_queries_no_metrics( def test_load_queries_no_databases( self, - logger: logging.Logger, - config_full: dict[str, Any], + config_full: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - """An error is raised if no databases are configured.""" config_full["queries"]["q"]["databases"] = [] config_file = write_config(config_full) - with pytest.raises(ConfigError) as err, config_file.open() as fd: - load_config(fd, logger) + with pytest.raises(ConfigError) as err: + load_config(config_file) assert ( str(err.value) == "Invalid config at queries/q/databases: [] should be non-empty" @@ -1004,23 +927,19 @@ def test_load_queries_no_databases( ) def test_load_metrics_expiration( self, - logger: logging.Logger, - config_full: dict[str, Any], + config_full: dict[str, t.Any], write_config: ConfigWriter, expiration: str | int | None, value: int | None, ) -> None: - """The metric series expiration time can be specified with suffixes.""" config_full["metrics"]["m"]["expiration"] = expiration config_file = write_config(config_full) - with config_file.open() as fd: - config = load_config(fd, logger) + config = load_config(config_file) assert config.metrics["m"].config["expiration"] == value class TestResolveDSN: def test_all_details(self) -> None: - """The DSN can be specified as a dictionary.""" details = { "dialect": "postgresql", "user": "user", @@ -1036,7 +955,6 @@ def test_all_details(self) -> None: ) def test_db_as_path(self) -> None: - """If the db name is a path, it's treated accordingly.""" details = { "dialect": "sqlite", "database": "/path/to/file", @@ -1044,7 +962,6 @@ def test_db_as_path(self) -> None: assert _resolve_dsn(details, {}) == "sqlite:///path/to/file" def test_encode_user_password(self) -> None: - """The user and password are encoded.""" details = { "dialect": "postgresql", "user": "us%r", @@ -1058,7 +975,6 @@ def test_encode_user_password(self) -> None: ) def test_encode_options(self) -> None: - """Option parmaeters are encoded.""" details = { "dialect": "postgresql", "database": "/mydb", @@ -1075,7 +991,7 @@ def test_encode_options(self) -> None: class TestGetParametersSets: def test_list(self) -> None: - params: list[dict[str, Any]] = [ + params: list[dict[str, t.Any]] = [ { "param1": 100, "param2": "foo", @@ -1088,7 +1004,7 @@ def test_list(self) -> None: assert list(_get_parameters_sets(params)) == params def test_dict(self) -> None: - params: dict[str, list[dict[str, Any]]] = { + params: dict[str, list[dict[str, t.Any]]] = { "param1": [ { "sub1": 100, diff --git a/tests/conftest.py b/tests/conftest.py index b56b77c..5994bdd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,9 @@ import asyncio from collections.abc import AsyncIterator, Iterator -import re import pytest from pytest_mock import MockerFixture +from pytest_structlog import StructuredLogCapture from toolrack.testing.fixtures import advance_time from query_exporter.db import DataBase, MetricResults, Query @@ -11,6 +11,12 @@ __all__ = ["advance_time", "query_tracker"] +@pytest.fixture(autouse=True) +def _autouse(log: StructuredLogCapture) -> Iterator[None]: + """Autouse dependent fixtures.""" + yield None + + class QueryTracker: def __init__(self) -> None: self.queries: list[Query] = [] @@ -57,22 +63,3 @@ async def execute(db: DataBase, query: Query) -> MetricResults: mocker.patch.object(DataBase, "execute", execute) yield tracker - - -class AssertRegexpMatch: - """Assert that comparison matches the specified regexp.""" - - def __init__(self, pattern: str, flags: int = 0) -> None: - self._re = re.compile(pattern, flags) - - def __eq__(self, string: object) -> bool: - return bool(self._re.match(str(string))) - - def __repr__(self) -> str: - return self._re.pattern # pragma: nocover - - -@pytest.fixture -def re_match() -> Iterator[type[AssertRegexpMatch]]: - """Matcher for asserting that a string matches a regexp.""" - yield AssertRegexpMatch diff --git a/tests/db_test.py b/tests/db_test.py index ae0d504..0b82dc8 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -1,9 +1,12 @@ import asyncio from collections.abc import Iterator -import logging import time +import typing as t +from unittest.mock import ANY import pytest +from pytest_mock import MockerFixture +from pytest_structlog import StructuredLogCapture from sqlalchemy import ( create_engine, text, @@ -12,10 +15,11 @@ Connection, Engine, ) +from sqlalchemy.sql.elements import TextClause -from query_exporter.config import DataBaseConfig from query_exporter.db import ( DataBase, + DataBaseConfig, DataBaseConnectError, DataBaseConnection, DataBaseError, @@ -35,34 +39,26 @@ class TestInvalidResultCount: - def test_message(self): - """The error messagecontains counts.""" + def test_message(self) -> None: error = InvalidResultCount(1, 2) assert str(error) == "Wrong result count from query: expected 1, got 2" class TestCreateDBEngine: - def test_instantiate_missing_engine_module(self, caplog): - """An error is raised if a module for the engine is missing.""" - with caplog.at_level(logging.ERROR): - with pytest.raises(DataBaseError) as error: - create_db_engine("postgresql:///foo") + def test_instantiate_missing_engine_module(self) -> None: + with pytest.raises(DataBaseError) as error: + create_db_engine("postgresql:///foo") assert str(error.value) == 'module "psycopg2" not found' @pytest.mark.parametrize("dsn", ["foo-bar", "unknown:///db"]) - def test_instantiate_invalid_dsn(self, caplog, dsn): - """An error is raised if a the provided DSN is invalid.""" - with ( - caplog.at_level(logging.ERROR), - pytest.raises(DataBaseError) as error, - ): + def test_instantiate_invalid_dsn(self, dsn: str) -> None: + with pytest.raises(DataBaseError) as error: create_db_engine(dsn) assert str(error.value) == f'Invalid database DSN: "{dsn}"' class TestQuery: - def test_instantiate(self): - """A query can be instantiated with the specified arguments.""" + def test_instantiate(self) -> None: query = Query( "query", ["db1", "db2"], @@ -84,8 +80,7 @@ def test_instantiate(self): assert query.interval is None assert query.timeout is None - def test_instantiate_with_config_name(self): - """A query can be instantiated with a config_name different from name.""" + def test_instantiate_with_config_name(self) -> None: query = Query( "query", ["db"], @@ -95,8 +90,7 @@ def test_instantiate_with_config_name(self): ) assert query.config_name == "query_config" - def test_instantiate_with_parameters(self): - """A query can be instantiated with parameters.""" + def test_instantiate_with_parameters(self) -> None: query = Query( "query", ["db1", "db2"], @@ -110,8 +104,7 @@ def test_instantiate_with_parameters(self): ) assert query.parameters == {"param1": 1, "param2": 2} - def test_instantiate_parameters_not_matching(self): - """If parameters don't match those in SQL, an error is raised.""" + def test_instantiate_parameters_not_matching(self) -> None: with pytest.raises(InvalidQueryParameters): Query( "query", @@ -125,8 +118,7 @@ def test_instantiate_parameters_not_matching(self): parameters={"param1": 1, "param2": 2}, ) - def test_instantiate_with_interval(self): - """A query can be instantiated with an interval.""" + def test_instantiate_with_interval(self) -> None: query = Query( "query", ["db1", "db2"], @@ -139,8 +131,7 @@ def test_instantiate_with_interval(self): ) assert query.interval == 20 - def test_instantiate_with_schedule(self): - """A query can be instantiated with a schedule.""" + def test_instantiate_with_schedule(self) -> None: query = Query( "query", ["db1", "db2"], @@ -153,8 +144,7 @@ def test_instantiate_with_schedule(self): ) assert query.schedule == "0 * * * *" - def test_instantiate_with_interval_and_schedule(self): - """Interval and schedule can't be specified together.""" + def test_instantiate_with_interval_and_schedule(self) -> None: with pytest.raises(InvalidQuerySchedule) as error: Query( "query", @@ -169,8 +159,7 @@ def test_instantiate_with_interval_and_schedule(self): == 'Invalid schedule for query "query": both interval and schedule specified' ) - def test_instantiate_with_invalid_schedule(self): - """Invalid query schedule raises an error.""" + def test_instantiate_with_invalid_schedule(self) -> None: with pytest.raises(InvalidQuerySchedule) as error: Query( "query", @@ -184,8 +173,7 @@ def test_instantiate_with_invalid_schedule(self): == 'Invalid schedule for query "query": invalid schedule format' ) - def test_instantiate_with_timeout(self): - """A query can be instantiated with a timeout.""" + def test_instantiate_with_timeout(self) -> None: query = Query( "query", ["db"], @@ -203,8 +191,7 @@ def test_instantiate_with_timeout(self): ({"schedule": "1 * * * *"}, True), ], ) - def test_timed(self, kwargs, is_timed): - """Query.timed reports whether the query is run with a time schedule""" + def test_timed(self, kwargs: dict[str, t.Any], is_timed: bool) -> None: query = Query( "query", ["db1", "db2"], @@ -217,8 +204,7 @@ def test_timed(self, kwargs, is_timed): ) assert query.timed == is_timed - def test_labels(self): - """All labels for the query can be returned.""" + def test_labels(self) -> None: query = Query( "query", ["db1", "db2"], @@ -230,15 +216,13 @@ def test_labels(self): ) assert query.labels() == frozenset(["label1", "label2"]) - def test_results_empty(self): - """No error is raised if the result set is empty""" + def test_results_empty(self) -> None: query = Query("query", ["db"], [QueryMetric("metric", [])], "") query_results = QueryResults(["one"], []) metrics_results = query.results(query_results) assert metrics_results.results == [] - def test_results_metrics(self): - """The results method returns results by matching metrics name.""" + def test_results_metrics(self) -> None: query = Query( "query", ["db"], @@ -256,8 +240,7 @@ def test_results_metrics(self): MetricResult("metric2", 33, {}), ] - def test_results_metrics_with_labels(self): - """The results method returns results by matching metrics name.""" + def test_results_metrics_with_labels(self) -> None: query = Query( "query", ["db"], @@ -279,15 +262,13 @@ def test_results_metrics_with_labels(self): MetricResult("metric2", 33, {"label2": "baz"}), ] - def test_results_wrong_result_count(self): - """An error is raised if the result column count is wrong.""" + def test_results_wrong_result_count(self) -> None: query = Query("query", ["db"], [QueryMetric("metric1", [])], "") query_results = QueryResults(["one", "two"], [(1, 2)]) with pytest.raises(InvalidResultCount): query.results(query_results) - def test_results_wrong_result_count_with_label(self): - """An error is raised if the result column count is wrong.""" + def test_results_wrong_result_count_with_label(self) -> None: query = Query( "query", ["db"], [QueryMetric("metric1", ["label1"])], "" ) @@ -295,8 +276,7 @@ def test_results_wrong_result_count_with_label(self): with pytest.raises(InvalidResultCount): query.results(query_results) - def test_results_wrong_names_with_labels(self): - """An error is raised if metric and labels names don't match.""" + def test_results_wrong_names_with_labels(self) -> None: query = Query( "query", ["db"], [QueryMetric("metric1", ["label1"])], "" ) @@ -310,8 +290,7 @@ def test_results_wrong_names_with_labels(self): class TestQueryResults: - def test_from_result(self): - """The from_result method returns a QueryResult.""" + def test_from_result(self) -> None: engine = create_engine("sqlite://") with engine.connect() as conn: result = conn.execute(text("SELECT 1 AS a, 2 AS b")) @@ -321,8 +300,7 @@ def test_from_result(self): assert query_results.latency is None assert query_results.timestamp < time.time() - def test_from_empty(self): - """The from_result method returns empty QueryResult.""" + def test_from_empty(self) -> None: engine = create_engine("sqlite://") with engine.connect() as conn: result = conn.execute(text("PRAGMA auto_vacuum = 1")) @@ -331,8 +309,7 @@ def test_from_empty(self): assert query_results.rows == [] assert query_results.latency is None - def test_from_result_with_latency(self): - """The from_result method tracks call latency.""" + def test_from_result_with_latency(self) -> None: engine = create_engine("sqlite://") with engine.connect() as conn: result = conn.execute(text("SELECT 1 AS a, 2 AS b")) @@ -354,7 +331,7 @@ async def conn() -> Iterator[DataBaseConnection]: class TestWorkerAction: - async def test_call_wait(self): + async def test_call_wait(self) -> None: def func(a: int, b: int) -> int: return a + b @@ -362,7 +339,7 @@ def func(a: int, b: int) -> int: action() assert await action.result() == 30 - async def test_call_exception(self): + async def test_call_exception(self) -> None: def func() -> None: raise Exception("fail!") @@ -374,46 +351,39 @@ def func() -> None: class TestDataBaseConnection: - def test_engine(self, conn): - """The connection keeps the SQLAlchemy engine.""" + def test_engine(self, conn: DataBaseConnection) -> None: assert isinstance(conn.engine, Engine) async def test_open(self, conn: DataBaseConnection) -> None: - """The open method opens the database connection.""" await conn.open() assert conn.connected assert conn._conn is not None assert conn._worker.is_alive() async def test_open_noop(self, conn: DataBaseConnection) -> None: - """The open method is a no-op if connection is already open.""" await conn.open() await conn.open() assert conn.connected async def test_close(self, conn: DataBaseConnection) -> None: - """The close method closes the connection.""" await conn.open() await conn.close() assert not conn.connected assert conn._conn is None async def test_close_noop(self, conn: DataBaseConnection) -> None: - """The close method is a no-op if connection is already closed.""" await conn.open() await conn.close() await conn.close() assert not conn.connected async def test_execute(self, conn: DataBaseConnection) -> None: - """The connection can execute queries.""" await conn.open() query_results = await conn.execute(text("SELECT 1 AS a, 2 AS b")) assert query_results.keys == ["a", "b"] assert query_results.rows == [(1, 2)] async def test_execute_with_params(self, conn: DataBaseConnection) -> None: - """The connection can execute queries with parameters.""" await conn.open() query_results = await conn.execute( text("SELECT :a AS a, :b AS b"), parameters={"a": 1, "b": 2} @@ -423,67 +393,60 @@ async def test_execute_with_params(self, conn: DataBaseConnection) -> None: @pytest.fixture -def db_config(): - return DataBaseConfig( +def db_config() -> Iterator[DataBaseConfig]: + yield DataBaseConfig( name="db", dsn="sqlite://", ) @pytest.fixture -async def db(db_config): +async def db(db_config: DataBaseConfig) -> Iterator[DataBase]: db = DataBase(db_config) yield db await db.close() class TestDataBase: - def test_instantiate(self, db_config): - """A DataBase can be instantiated with the specified arguments.""" - db = DataBase(db_config) - assert db.config is db_config - assert db.logger == logging.getLogger() - - async def test_as_context_manager(self, db): - """The database can be used as an async context manager.""" + async def test_as_context_manager(self, db: DataBase) -> None: async with db: query_result = await db.execute_sql("SELECT 10 AS a, 20 AS b") assert query_result.rows == [(10, 20)] # the db is closed at context exit assert not db.connected - async def test_connect(self, caplog, re_match, db): - """The connect connects to the database.""" - with caplog.at_level(logging.DEBUG): - await db.connect() + async def test_connect(self, db: DataBase) -> None: + await db.connect() assert db.connected assert isinstance(db._conn._conn, Connection) - assert caplog.messages == [ - re_match(r'worker "DataBase-db": started'), - 'worker "DataBase-db": received action "_connect"', - 'connected to database "db"', - ] - async def test_connect_lock(self, caplog, re_match, db): - """The connect method has a lock to prevent concurrent calls.""" - with caplog.at_level(logging.DEBUG): - await asyncio.gather(db.connect(), db.connect()) - assert caplog.messages == [ - re_match(r'worker "DataBase-db": started'), - 'worker "DataBase-db": received action "_connect"', - 'connected to database "db"', - ] + async def test_connect_log( + self, log: StructuredLogCapture, db: DataBase + ) -> None: + await db.connect() + assert log.has("start", database="db", worker_id=ANY, level="debug") + assert log.has( + "action received", + action="_connect", + database="db", + worker_id=ANY, + level="debug", + ) + assert log.has( + "connected", database="db", worker_id=ANY, level="debug" + ) + + async def test_connect_lock(self, db: DataBase) -> None: + await asyncio.gather(db.connect(), db.connect()) - async def test_connect_error(self): - """A DataBaseConnectError is raised if database connection fails.""" + async def test_connect_error(self) -> None: config = DataBaseConfig(name="db", dsn="sqlite:////invalid") db = DataBase(config) with pytest.raises(DataBaseConnectError) as error: await db.connect() assert "unable to open database file" in str(error.value) - async def test_connect_sql(self): - """If connect_sql is specified, it's run at connection.""" + async def test_connect_sql(self) -> None: config = DataBaseConfig( name="db", dsn="sqlite://", @@ -501,38 +464,33 @@ async def execute_sql(sql): assert queries == ["SELECT 1", "SELECT 2"] await db.close() - async def test_connect_sql_fail(self, caplog): - """If the SQL at connection fails, an error is raised.""" + async def test_connect_sql_fail(self, log: StructuredLogCapture) -> None: config = DataBaseConfig( name="db", dsn="sqlite://", connect_sql=["WRONG"], ) db = DataBase(config) - with ( - caplog.at_level(logging.DEBUG), - pytest.raises(DataBaseQueryError) as error, - ): + with pytest.raises(DataBaseQueryError) as error: await db.connect() assert not db.connected assert 'failed executing query "WRONG"' in str(error.value) - assert 'disconnected from database "db"' in caplog.messages + assert log.has("disconnected", database="db") - async def test_close(self, caplog, re_match, db): - """The close method closes database connection.""" + async def test_close( + self, log: StructuredLogCapture, db: DataBase + ) -> None: await db.connect() - with caplog.at_level(logging.DEBUG): - await db.close() - assert caplog.messages == [ - 'worker "DataBase-db": received action "_close"', - 'worker "DataBase-db": shutting down', - 'disconnected from database "db"', - ] + await db.close() + assert log.has("action received", worker_id=ANY, action="_close") + assert log.has("shutdown", worker_id=ANY) + assert log.has("disconnected", database="db") assert not db.connected assert db._conn._conn is None - async def test_execute_log(self, db, caplog): - """A message is logged about the query being executed.""" + async def test_execute_log( + self, log: StructuredLogCapture, db: DataBase + ) -> None: query = Query( "query", ["db"], @@ -540,18 +498,16 @@ async def test_execute_log(self, db, caplog): "SELECT 1.0 AS metric", ) await db.connect() - with caplog.at_level(logging.DEBUG): - await db.execute(query) - assert caplog.messages == [ - 'running query "query" on database "db"', - 'worker "DataBase-db": received action "_execute"', - 'worker "DataBase-db": received action "from_result"', - ] + await db.execute(query) + assert log.has("run query", query="query", database="db") + assert log.has("action received", worker_id=ANY, action="_execute") + assert log.has("action received", worker_id=ANY, action="from_result") await db.close() @pytest.mark.parametrize("connected", [True, False]) - async def test_execute_keep_connected(self, mocker, connected): - """If keep_connected is set to true, the db is not closed.""" + async def test_execute_keep_connected( + self, mocker: MockerFixture, connected: bool + ) -> None: config = DataBaseConfig( name="db", dsn="sqlite://", keep_connected=connected ) @@ -570,8 +526,9 @@ async def test_execute_keep_connected(self, mocker, connected): mock_conn_detach.assert_called_once() await db.close() - async def test_execute_no_keep_disconnect_after_pending_queries(self): - """The db is disconnected only after pending queries are run.""" + async def test_execute_no_keep_disconnect_after_pending_queries( + self, + ) -> None: config = DataBaseConfig( name="db", dsn="sqlite://", keep_connected=False ) @@ -592,8 +549,7 @@ async def test_execute_no_keep_disconnect_after_pending_queries(self): await asyncio.gather(db.execute(query1), db.execute(query2)) assert not db.connected - async def test_execute_not_connected(self, db): - """The execute recconnects to the database if not connected.""" + async def test_execute_not_connected(self, db: DataBase) -> None: query = Query( "query", ["db"], [QueryMetric("metric", [])], "SELECT 1 AS metric" ) @@ -602,8 +558,7 @@ async def test_execute_not_connected(self, db): # the connection is kept for reuse assert db.connected - async def test_execute(self, db): - """The execute method executes a query.""" + async def test_execute(self, db: DataBase) -> None: sql = ( "SELECT * FROM (SELECT 10 AS metric1, 20 AS metric2 UNION" " SELECT 30 AS metric1, 40 AS metric2)" @@ -624,8 +579,7 @@ async def test_execute(self, db): ] assert isinstance(metric_results.latency, float) - async def test_execute_with_labels(self, db): - """The execute method executes a query with labels.""" + async def test_execute_with_labels(self, db: DataBase) -> None: sql = """ SELECT metric2, metric1, label2, label1 FROM ( SELECT 11 AS metric2, 22 AS metric1, @@ -653,16 +607,16 @@ async def test_execute_with_labels(self, db): MetricResult("metric2", 33, {"label2": "baz"}), ] - async def test_execute_fail(self, caplog, db): - """If the query fails, an exception is raised.""" + async def test_execute_fail(self, db: DataBase) -> None: query = Query("query", 10, [QueryMetric("metric", [])], "WRONG") await db.connect() with pytest.raises(DataBaseQueryError) as error: await db.execute(query) assert "syntax error" in str(error.value) - async def test_execute_query_invalid_count(self, caplog, db): - """If the number of fields don't match, an error is raised.""" + async def test_execute_query_invalid_count( + self, log: StructuredLogCapture, db: DataBase + ) -> None: query = Query( "query", 20, @@ -670,23 +624,24 @@ async def test_execute_query_invalid_count(self, caplog, db): "SELECT 1 AS metric, 2 AS other", ) await db.connect() - with ( - caplog.at_level(logging.ERROR), - pytest.raises(DataBaseQueryError) as error, - ): + with pytest.raises(DataBaseQueryError) as error: await db.execute(query) assert ( str(error.value) == "Wrong result count from query: expected 1, got 2" ) assert error.value.fatal - assert caplog.messages == [ - 'query "query" on database "db" failed: ' - "Wrong result count from query: expected 1, got 2" - ] + assert log.has( + "query failed", + level="error", + query="query", + database="db", + error="Wrong result count from query: expected 1, got 2", + ) - async def test_execute_query_invalid_count_with_labels(self, db): - """If the number of fields don't match, an error is raised.""" + async def test_execute_query_invalid_count_with_labels( + self, db: DataBase + ) -> None: query = Query( "query", ["db"], @@ -702,8 +657,9 @@ async def test_execute_query_invalid_count_with_labels(self, db): ) assert error.value.fatal - async def test_execute_invalid_names_with_labels(self, db): - """If the names of fields don't match, an error is raised.""" + async def test_execute_invalid_names_with_labels( + self, db: DataBase + ) -> None: query = Query( "query", ["db"], @@ -719,8 +675,9 @@ async def test_execute_invalid_names_with_labels(self, db): ) assert error.value.fatal - async def test_execute_traceback_debug(self, caplog, mocker, db): - """Traceback are logged as debug messages.""" + async def test_execute_debug_exception( + self, mocker: MockerFixture, log: StructuredLogCapture, db: DataBase + ) -> None: query = Query( "query", ["db"], @@ -728,22 +685,24 @@ async def test_execute_traceback_debug(self, caplog, mocker, db): "SELECT 1 AS metric", ) await db.connect() - mocker.patch.object(db, "execute_sql").side_effect = Exception("boom!") - with ( - caplog.at_level(logging.DEBUG), - pytest.raises(DataBaseQueryError) as error, - ): + exception = Exception("boom!") + mocker.patch.object(db, "execute_sql").side_effect = exception + + with pytest.raises(DataBaseQueryError) as error: await db.execute(query) assert str(error.value) == "boom!" assert not error.value.fatal - assert ( - 'query "query" on database "db" failed: boom!' in caplog.messages + assert log.has( + "query failed", + query="query", + database="db", + exception=exception, + level="error", ) - # traceback is included in messages - assert "await self.execute_sql(" in caplog.messages[-1] - async def test_execute_timeout(self, caplog, db): - """If the query times out, an error is raised and logged.""" + async def test_execute_timeout( + self, log: StructuredLogCapture, db: DataBase + ) -> None: query = Query( "query", ["db"], @@ -753,26 +712,24 @@ async def test_execute_timeout(self, caplog, db): ) await db.connect() - async def execute(sql, parameters): + async def execute( + sql: TextClause, + parameters: dict[str, t.Any] | None = None, + ) -> QueryResults: await asyncio.sleep(1) # longer than timeout db._conn.execute = execute - with ( - caplog.at_level(logging.WARNING), - pytest.raises(QueryTimeoutExpired) as error, - ): + with pytest.raises(QueryTimeoutExpired): await db.execute(query) - assert ( - str(error.value) - == 'Execution for query "query" expired after 0.1 seconds' + assert log.has( + "query timeout", + query="query", + database="db", + level="warning", ) - assert caplog.messages == [ - 'Execution for query "query" expired after 0.1 seconds' - ] - async def test_execute_sql(self, db): - """It's possible to execute raw SQL.""" + async def test_execute_sql(self, db: DataBase) -> None: await db.connect() result = await db.execute_sql("SELECT 10, 20") assert result.rows == [(10, 20)] @@ -788,5 +745,4 @@ async def test_execute_sql(self, db): def test_error_message( self, db: DataBase, error: str | Exception, message: str ) -> None: - """An error message is returned both for strings and exceptions.""" assert db._error_message(error) == message diff --git a/tests/loop_test.py b/tests/loop_test.py index a269ab2..171088d 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -2,12 +2,13 @@ from collections import defaultdict from collections.abc import Callable, Iterator from decimal import Decimal -import logging from pathlib import Path -from typing import Any +import typing as t +from unittest.mock import ANY from prometheus_aioexporter import MetricsRegistry import pytest +from pytest_structlog import StructuredLogCapture import yaml from query_exporter import loop @@ -19,7 +20,7 @@ @pytest.fixture -def config_data() -> Iterator[dict[str, Any]]: +def config_data() -> Iterator[dict[str, t.Any]]: yield { "databases": {"db": {"dsn": "sqlite://"}}, "metrics": {"m": {"type": "gauge"}}, @@ -41,18 +42,16 @@ def registry() -> Iterator[MetricsRegistry]: @pytest.fixture async def make_query_loop( - tmp_path: Path, config_data: dict[str, Any], registry: MetricsRegistry + tmp_path: Path, config_data: dict[str, t.Any], registry: MetricsRegistry ) -> Iterator[Callable[[], MetricsRegistry]]: query_loops = [] def make_loop() -> loop.QueryLoop: config_file = tmp_path / "config.yaml" config_file.write_text(yaml.dump(config_data), "utf-8") - logger = logging.getLogger() - with config_file.open() as fh: - config = load_config(fh, logger) + config = load_config(config_file) registry.create_metrics(config.metrics.values()) - query_loop = loop.QueryLoop(config, registry, logger) + query_loop = loop.QueryLoop(config, registry) query_loops.append(query_loop) return query_loop @@ -98,7 +97,6 @@ async def run_queries(db_file: Path, *queries: str) -> None: class TestMetricsLastSeen: def test_update(self) -> None: - """Last seen times are tracked for each series of metrics with expiration.""" last_seen = loop.MetricsLastSeen({"m1": 50, "m2": 100}) last_seen.update("m1", {"l1": "v1", "l2": "v2"}, 100) last_seen.update("m1", {"l1": "v3", "l2": "v4"}, 200) @@ -111,13 +109,11 @@ def test_update(self) -> None: } def test_update_label_values_sorted_by_name(self) -> None: - """Last values are sorted by label names.""" last_seen = loop.MetricsLastSeen({"m1": 50}) last_seen.update("m1", {"l2": "v2", "l1": "v1"}, 100) assert last_seen._last_seen == {"m1": {("v1", "v2"): 100}} def test_expire_series_not_expired(self) -> None: - """If no entry for a metric is expired, it's not returned.""" last_seen = loop.MetricsLastSeen({"m1": 50}) last_seen.update("m1", {"l1": "v1", "l2": "v2"}, 10) last_seen.update("m1", {"l1": "v3", "l2": "v4"}, 20) @@ -130,7 +126,6 @@ def test_expire_series_not_expired(self) -> None: } def test_expire_series(self) -> None: - """Expired metric series are returned and removed.""" last_seen = loop.MetricsLastSeen({"m1": 50, "m2": 100}) last_seen.update("m1", {"l1": "v1", "l2": "v2"}, 10) last_seen.update("m1", {"l1": "v3", "l2": "v4"}, 100) @@ -151,14 +146,12 @@ def test_expire_no_labels(self) -> None: class TestQueryLoop: async def test_start(self, query_tracker, query_loop) -> None: - """The start method starts timed calls for queries.""" await query_loop.start() timed_call = query_loop._timed_calls["q"] assert timed_call.running await query_tracker.wait_results() async def test_stop(self, query_loop) -> None: - """The stop method stops timed calls for queries.""" await query_loop.start() timed_call = query_loop._timed_calls["q"] await query_loop.stop() @@ -167,7 +160,6 @@ async def test_stop(self, query_loop) -> None: async def test_run_query( self, query_tracker, query_loop, registry ) -> None: - """Queries are run and update metrics.""" await query_loop.start() await query_tracker.wait_results() # the metric is updated @@ -187,8 +179,7 @@ async def test_run_scheduled_query( registry, config_data, make_query_loop, - ): - """Queries are run and update metrics.""" + ) -> None: event_loop = asyncio.get_running_loop() def croniter(*args): @@ -209,8 +200,7 @@ def croniter(*args): async def test_run_query_with_parameters( self, query_tracker, registry, config_data, make_query_loop - ): - """Queries are run with declared parameters.""" + ) -> None: config_data["metrics"]["m"]["type"] = "counter" config_data["metrics"]["m"]["labels"] = ["l"] config_data["queries"]["q"]["sql"] = "SELECT :param AS m, :label as l" @@ -235,8 +225,7 @@ async def test_run_query_with_parameters( async def test_run_query_null_value( self, query_tracker, registry, config_data, make_query_loop - ): - """A null value in query results is treated like a zero.""" + ) -> None: config_data["queries"]["q"]["sql"] = "SELECT NULL AS m" query_loop = make_query_loop() await query_loop.start() @@ -246,8 +235,7 @@ async def test_run_query_null_value( async def test_run_query_counter_no_increment( self, query_tracker, registry, config_data, make_query_loop - ): - """If increment is set to False, counter is set to the new value.""" + ) -> None: config_data["metrics"]["m"]["type"] = "counter" config_data["metrics"]["m"]["increment"] = False config_data["queries"]["q"]["sql"] = "SELECT :param AS m" @@ -264,8 +252,7 @@ async def test_run_query_counter_no_increment( async def test_run_query_metrics_with_database_labels( self, query_tracker, registry, config_data, make_query_loop - ): - """If databases have extra labels, they're set for metrics.""" + ) -> None: config_data["databases"] = { "db1": {"dsn": "sqlite://", "labels": {"l1": "v1", "l2": "v2"}}, "db2": {"dsn": "sqlite://", "labels": {"l1": "v3", "l2": "v4"}}, @@ -282,8 +269,7 @@ async def test_run_query_metrics_with_database_labels( async def test_update_metric_decimal_value( self, registry, make_query_loop - ): - """A Decimal value in query results is converted to float.""" + ) -> None: db = DataBase(DataBaseConfig(name="db", dsn="sqlite://")) query_loop = make_query_loop() query_loop._update_metric(db, "m", Decimal("100.123")) @@ -293,60 +279,61 @@ async def test_update_metric_decimal_value( assert isinstance(value, float) async def test_run_query_log( - self, caplog, re_match, query_tracker, query_loop - ): - """Debug messages are logged on query execution.""" - caplog.set_level(logging.DEBUG) + self, + log: StructuredLogCapture, + query_tracker, + query_loop, + ) -> None: await query_loop.start() await query_tracker.wait_queries() - assert caplog.messages == [ - re_match(r'worker "DataBase-db": started'), - 'worker "DataBase-db": received action "_connect"', - 'connected to database "db"', - 'running query "q" on database "db"', - 'worker "DataBase-db": received action "_execute"', - 'worker "DataBase-db": received action "from_result"', - 'updating metric "m" set 100.0 {database="db"}', - re_match( - r'updating metric "query_latency" observe .* \{database="db",query="q"\}' + assert [ + log.debug( + "updating metric", + metric="m", + method="set", + value=100.0, + labels={"database": "db"}, ), - re_match( - r'updating metric "query_timestamp" set .* \{database="db",query="q"\}' + log.debug( + "updating metric", + metric="query_latency", + method="observe", + value=ANY, + labels={"database": "db", "query": "q"}, ), - 'updating metric "queries" inc 1 {database="db",query="q",status="success"}', - ] + log.debug( + "updating metric", + metric="queries", + method="inc", + value=1, + labels={"database": "db", "query": "q", "status": "success"}, + ), + ] <= log.events async def test_run_query_log_labels( - self, caplog, re_match, query_tracker, config_data, make_query_loop - ): - """Debug messages include metric labels.""" + self, + log: StructuredLogCapture, + query_tracker, + config_data, + make_query_loop, + ) -> None: config_data["metrics"]["m"]["labels"] = ["l"] config_data["queries"]["q"]["sql"] = 'SELECT 100.0 AS m, "foo" AS l' query_loop = make_query_loop() - caplog.set_level(logging.DEBUG) await query_loop.start() await query_tracker.wait_queries() - assert caplog.messages == [ - re_match(r'worker "DataBase-db": started'), - 'worker "DataBase-db": received action "_connect"', - 'connected to database "db"', - 'running query "q" on database "db"', - 'worker "DataBase-db": received action "_execute"', - 'worker "DataBase-db": received action "from_result"', - 'updating metric "m" set 100.0 {database="db",l="foo"}', - re_match( - r'updating metric "query_latency" observe .* \{database="db",query="q"\}' - ), - re_match( - r'updating metric "query_timestamp" set .* \{database="db",query="q"\}' - ), - 'updating metric "queries" inc 1 {database="db",query="q",status="success"}', - ] + assert log.has( + "updating metric", + level="debug", + metric="m", + method="set", + value=100.0, + labels={"database": "db", "l": "foo"}, + ) async def test_run_query_increase_db_error_count( self, query_tracker, config_data, make_query_loop, registry - ): - """Query errors are logged.""" + ) -> None: config_data["databases"]["db"]["dsn"] = "sqlite:////invalid" query_loop = make_query_loop() await query_loop.start() @@ -356,8 +343,7 @@ async def test_run_query_increase_db_error_count( async def test_run_query_increase_database_error_count( self, mocker, query_tracker, config_data, make_query_loop, registry - ): - """Count of database errors is incremented on failed connection.""" + ) -> None: query_loop = make_query_loop() db = query_loop._databases["db"] mock_connect = mocker.patch.object(db._conn.engine, "connect") @@ -369,8 +355,7 @@ async def test_run_query_increase_database_error_count( async def test_run_query_increase_query_error_count( self, query_tracker, config_data, make_query_loop, registry - ): - """Count of errored queries is incremented on error.""" + ) -> None: config_data["queries"]["q"]["sql"] = "SELECT 100.0 AS a, 200.0 AS b" query_loop = make_query_loop() await query_loop.start() @@ -382,8 +367,7 @@ async def test_run_query_increase_query_error_count( async def test_run_query_increase_timeout_count( self, query_tracker, config_data, make_query_loop, registry - ): - """Count of errored queries is incremented on timeout.""" + ) -> None: config_data["queries"]["q"]["timeout"] = 0.1 query_loop = make_query_loop() await query_loop.start() @@ -403,8 +387,7 @@ async def execute(sql, parameters): async def test_run_query_at_interval( self, advance_time, query_tracker, query_loop - ): - """Queries are run at the specified time interval.""" + ) -> None: await query_loop.start() await advance_time(0) # kick the first run # the query has been run once @@ -418,8 +401,7 @@ async def test_run_query_at_interval( async def test_run_timed_queries_invalid_result_count( self, query_tracker, config_data, make_query_loop - ): - """Timed queries returning invalid elements count are removed.""" + ) -> None: config_data["queries"]["q"]["sql"] = "SELECT 100.0 AS a, 200.0 AS b" config_data["queries"]["q"]["interval"] = 1.0 query_loop = make_query_loop() @@ -438,8 +420,7 @@ async def test_run_timed_queries_invalid_result_count( async def test_run_timed_queries_invalid_result_count_stop_task( self, query_tracker, config_data, make_query_loop - ): - """Timed queries returning invalid result counts are stopped.""" + ) -> None: config_data["queries"]["q"]["sql"] = "SELECT 100.0 AS a, 200.0 AS b" config_data["queries"]["q"]["interval"] = 1.0 query_loop = make_query_loop() @@ -453,8 +434,7 @@ async def test_run_timed_queries_invalid_result_count_stop_task( async def test_run_timed_queries_not_removed_if_not_failing_on_all_dbs( self, tmp_path, query_tracker, config_data, make_query_loop - ): - """Timed queries are removed when they fail on all databases.""" + ) -> None: db1 = tmp_path / "db1.sqlite" db2 = tmp_path / "db2.sqlite" config_data["databases"] = { @@ -493,8 +473,7 @@ async def test_run_timed_queries_not_removed_if_not_failing_on_all_dbs( async def test_run_aperiodic_queries( self, query_tracker, config_data, make_query_loop - ): - """Queries with null interval can be run explicitly.""" + ) -> None: del config_data["queries"]["q"]["interval"] query_loop = make_query_loop() await query_loop.run_aperiodic_queries() @@ -504,8 +483,7 @@ async def test_run_aperiodic_queries( async def test_run_aperiodic_queries_invalid_result_count( self, query_tracker, config_data, make_query_loop - ): - """Aperiodic queries returning invalid elements count are removed.""" + ) -> None: config_data["queries"]["q"]["sql"] = "SELECT 100.0 AS a, 200.0 AS b" del config_data["queries"]["q"]["interval"] query_loop = make_query_loop() @@ -518,8 +496,7 @@ async def test_run_aperiodic_queries_invalid_result_count( async def test_run_aperiodic_queries_not_removed_if_not_failing_on_all_dbs( self, tmp_path, query_tracker, config_data, make_query_loop - ): - """Periodic queries are removed when they fail on all databases.""" + ) -> None: db1 = tmp_path / "db1.sqlite" db2 = tmp_path / "db2.sqlite" config_data["databases"] = { @@ -563,8 +540,7 @@ async def test_clear_expired_series( config_data, make_query_loop, registry, - ): - """The query loop clears out series last seen earlier than the specified interval.""" + ) -> None: db = tmp_path / "db.sqlite" config_data["databases"]["db"]["dsn"] = f"sqlite:///{db}" config_data["metrics"]["m"].update( From 928c510380f318464e8fbafe09c941ee8631da52 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sun, 29 Dec 2024 10:06:19 +0100 Subject: [PATCH 055/110] change "increment" default value for counters to False --- README.rst | 8 +++----- query_exporter/config.py | 2 ++ query_exporter/loop.py | 2 +- query_exporter/schemas/config.yaml | 10 +++------- tests/loop_test.py | 21 +++++++++++++++++---- 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/README.rst b/README.rst index bbfa763..91092f2 100644 --- a/README.rst +++ b/README.rst @@ -123,7 +123,7 @@ Each database definitions can have the following keys: All entries are optional, except ``dialect``. - Note that in the string form, username, password and options need to be + **Note**: in the string form, username, password and options need to be URL-encoded, whereas this is done automatically for the key/value form. See `database-specific options`_ page for some extra details on database @@ -215,12 +215,10 @@ Each metric definition can have the following keys: for ``counter`` metrics, whether to increment the value by the query result, or set the value to it. - By default, counters are incremented by the value returned by the query. If - this is set to ``false``, instead, the metric value will be set to the result + By default, counters are set to the value returned by the query. If this is + set to ``true``, instead, the metric value will be incremented by the result of the query. - **NOTE**: The default will be reversed in the 3.0 release, and ``increment`` - will be set to ``false`` by default. ``queries`` section ~~~~~~~~~~~~~~~~~~~ diff --git a/query_exporter/config.py b/query_exporter/config.py index 839e19d..ed80880 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -35,6 +35,7 @@ name=DB_ERRORS_METRIC_NAME, description="Number of database errors", type="counter", + config={"increment": True}, ) # metric for counting performed queries @@ -44,6 +45,7 @@ description="Number of database queries", type="counter", labels=("query", "status"), + config={"increment": True}, ) # metric for tracking last query execution timestamp QUERY_TIMESTAMP_METRIC_NAME = "query_timestamp" diff --git a/query_exporter/loop.py b/query_exporter/loop.py index bc8c582..839d86a 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -272,7 +272,7 @@ def _update_metric( def _get_metric_method(self, metric: MetricConfig) -> str: if metric.type == "counter" and not metric.config.get( - "increment", True + "increment", False ): method = "set" else: diff --git a/query_exporter/schemas/config.yaml b/query_exporter/schemas/config.yaml index d655854..e03b82a 100644 --- a/query_exporter/schemas/config.yaml +++ b/query_exporter/schemas/config.yaml @@ -195,15 +195,11 @@ definitions: increment: title: For counter metrics, whether to increment by query result description: > - If set to false, the counter value is set to the result of the query. + If set to true, the counter value is incremented by the result of the query. - By default, counters are incremented by the value returned by the - query. - - NOTE: This behavior will be reversed in the 3.0 release, and - increment will be set to false by default. + By default, counters are set to the value returned by the query. type: boolean - default: true + default: false query: title: Definition for a SQL query to run diff --git a/tests/loop_test.py b/tests/loop_test.py index 171088d..6964d8b 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -233,12 +233,25 @@ async def test_run_query_null_value( metric = registry.get_metric("m") assert metric_values(metric) == [0] - async def test_run_query_counter_no_increment( - self, query_tracker, registry, config_data, make_query_loop + @pytest.mark.parametrize( + "increment,value", + [ + (True, 30.0), + (False, 20.0), + ], + ) + async def test_run_query_counter( + self, + query_tracker, + registry, + config_data, + make_query_loop, + increment: bool, + value: float, ) -> None: config_data["metrics"]["m"]["type"] = "counter" - config_data["metrics"]["m"]["increment"] = False config_data["queries"]["q"]["sql"] = "SELECT :param AS m" + config_data["metrics"]["m"]["increment"] = increment config_data["queries"]["q"]["parameters"] = [ {"param": 10.0}, {"param": 20.0}, @@ -248,7 +261,7 @@ async def test_run_query_counter_no_increment( await query_tracker.wait_results() # the metric is updated metric = registry.get_metric("m") - assert metric_values(metric) == [20.0] + assert metric_values(metric) == [value] async def test_run_query_metrics_with_database_labels( self, query_tracker, registry, config_data, make_query_loop From 1832b08b4ebfd4afab99af5be651293bcd1f8a0d Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 27 Dec 2024 10:07:16 +0100 Subject: [PATCH 056/110] update dependencies --- requirements.txt | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index a0e996a..72af8c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,13 +12,13 @@ aiohttp==3.11.11 # query-exporter (pyproject.toml) aiosignal==1.3.2 # via aiohttp -argcomplete==3.5.2 - # via query-exporter (pyproject.toml) attrs==24.3.0 # via # aiohttp # jsonschema # referencing +click==8.1.8 + # via prometheus-aioexporter croniter==6.0.0 # via query-exporter (pyproject.toml) frozenlist==1.5.0 @@ -35,6 +35,10 @@ jsonschema==4.23.0 # via query-exporter (pyproject.toml) jsonschema-specifications==2024.10.1 # via jsonschema +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py multidict==6.1.0 # via # aiohttp @@ -43,7 +47,7 @@ packaging==24.2 # via pytest pluggy==1.5.0 # via pytest -prometheus-aioexporter==2.1.0 +prometheus-aioexporter==3.0.1 # via query-exporter (pyproject.toml) prometheus-client==0.21.1 # via @@ -53,12 +57,16 @@ propcache==0.2.1 # via # aiohttp # yarl +pygments==2.18.0 + # via rich pytest==8.3.4 # via toolrack python-dateutil==2.9.0.post0 # via # croniter # query-exporter (pyproject.toml) +python-dotenv==1.0.1 + # via prometheus-aioexporter pytz==2024.2 # via croniter pyyaml==6.0.2 @@ -67,6 +75,8 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications +rich==13.9.4 + # via prometheus-aioexporter rpds-py==0.22.3 # via # jsonschema @@ -75,10 +85,12 @@ six==1.17.0 # via python-dateutil sqlalchemy==2.0.36 # via query-exporter (pyproject.toml) -toolrack==4.0.1 +structlog==24.4.0 # via # prometheus-aioexporter # query-exporter (pyproject.toml) +toolrack==4.0.1 + # via query-exporter (pyproject.toml) typing-extensions==4.12.2 # via sqlalchemy yarl==1.18.3 From 25f280494af63731a77c1f18702b538cbcd36fd0 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 12 Dec 2024 13:38:08 +0100 Subject: [PATCH 057/110] docker/snap changes --- Dockerfile | 6 +++-- README.rst | 34 +++++++++++++++++++--------- snap/local/daemon.sh | 6 ++--- snap/local/query-exporter.completion | 32 -------------------------- snap/snapcraft.yaml | 6 +++++ 5 files changed, 35 insertions(+), 49 deletions(-) delete mode 100644 snap/local/query-exporter.completion diff --git a/Dockerfile b/Dockerfile index 11d4fed..91391cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,8 +62,10 @@ COPY --from=build-image /opt /opt ENV PATH="/virtualenv/bin:$PATH" ENV VIRTUAL_ENV="/virtualenv" ENV LD_LIBRARY_PATH="/opt/oracle/instantclient" +# IPv6 support is not enabled by default, only bind IPv4 +ENV QE_HOST="0.0.0.0" EXPOSE 9560/tcp VOLUME /config -# IPv6 support is not enabled by default, only bind IPv4 -ENTRYPOINT ["query-exporter", "/config/config.yaml", "-H", "0.0.0.0"] +WORKDIR /config +ENTRYPOINT ["query-exporter"] diff --git a/README.rst b/README.rst index 91092f2..43fff69 100644 --- a/README.rst +++ b/README.rst @@ -15,10 +15,13 @@ Each query can be run on multiple databases, and update multiple metrics. The application is simply run as:: - query-exporter config.yaml + query-exporter -where the passed configuration file contains the definitions of the databases -to connect and queries to perform to update metrics. +which will look for a ``config.yaml`` configuration file in the current +directory, containing the definitions of the databases to connect and queries +to perform to update metrics. The configuration file can be overridden by +passing the ``--config`` option, or setting the ``QE_CONFIG`` environment +variable. Configuration file format @@ -502,8 +505,9 @@ included by passing ``--process-stats`` in the command line. Debugging / Logs ---------------- -You can enable extended logging using the ``-L`` commandline switch. Possible -log levels are ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. +You can enable extended logging using the ``-L`` (or ``-log-level``) command +line switch. Possible log levels are ``critical``, ``error``, ``warning``, +``info``, ``debug``. Database engines @@ -537,6 +541,8 @@ To configure the daemon: - create or edit ``/var/snap/query-exporter/current/config.yaml`` with the configuration +- optionally, create a ``/var/snap/query-exporter/current/.env`` file with + environment variables definitions for additional config options - run ``sudo snap restart query-exporter`` The snap has support for connecting the following databases: @@ -561,12 +567,10 @@ where ``$CONFIG_DIR`` is the absolute path of a directory containing a ``config.yaml`` file, the configuration file to use. Alternatively, a volume name can be specified. - -A different ODBC driver version to use can be specified during image building, -by passing ``--build-arg ODBC_bVERSION_NUMBER``, e.g.:: - - docker build . --build-arg ODBC_DRIVER_VERSION=17 - +If, a ``.env`` file is present in the specified volume for ``/config``, its +content is loaded and applied to the environment for the exporter. The location +of the dotenv file can be customized by setting the ``QE_DOTENV`` environment +variable. The image has support for connecting the following databases: @@ -580,6 +584,14 @@ The image has support for connecting the following databases: A `Helm chart`_ to run the container in Kubernetes is also available. +ODBC driver version +~~~~~~~~~~~~~~~~~~~ + +A different ODBC driver version to use can be specified during image building, +by passing ``--build-arg ODBC_bVERSION_NUMBER``, e.g.:: + + docker build . --build-arg ODBC_DRIVER_VERSION=17 + .. _Prometheus: https://prometheus.io/ .. _SQLAlchemy: https://www.sqlalchemy.org/ diff --git a/snap/local/daemon.sh b/snap/local/daemon.sh index 1afb69d..8ad5767 100755 --- a/snap/local/daemon.sh +++ b/snap/local/daemon.sh @@ -1,7 +1,5 @@ #!/bin/sh -e -CONFIG_FILE="$SNAP_DATA/config.yaml" +[ -r "$SNAP_DATA/config.yaml" ] || snapctl stop "$SNAP_INSTANCE_NAME" -[ -r "$CONFIG_FILE" ] || snapctl stop "$SNAP_INSTANCE_NAME" - -exec "$SNAP/bin/query-exporter" -H 0.0.0.0 :: -- "$CONFIG_FILE" +exec env -C "$SNAP_DATA" "$SNAP/bin/query-exporter" -- diff --git a/snap/local/query-exporter.completion b/snap/local/query-exporter.completion deleted file mode 100644 index 6b5575a..0000000 --- a/snap/local/query-exporter.completion +++ /dev/null @@ -1,32 +0,0 @@ - -# Run something, muting output or redirecting it to the debug stream -# depending on the value of _ARC_DEBUG. -__python_argcomplete_run() { - if [[ -z "$_ARC_DEBUG" ]]; then - "$@" 8>&1 9>&2 1>/dev/null 2>&1 - else - "$@" 8>&1 9>&2 1>&9 2>&1 - fi -} - -_python_argcomplete() { - local IFS=$'\013' - local SUPPRESS_SPACE=0 - if compopt +o nospace 2> /dev/null; then - SUPPRESS_SPACE=1 - fi - COMPREPLY=( $(IFS="$IFS" \ - COMP_LINE="$COMP_LINE" \ - COMP_POINT="$COMP_POINT" \ - COMP_TYPE="$COMP_TYPE" \ - _ARGCOMPLETE_COMP_WORDBREAKS="$COMP_WORDBREAKS" \ - _ARGCOMPLETE=1 \ - _ARGCOMPLETE_SUPPRESS_SPACE=$SUPPRESS_SPACE \ - __python_argcomplete_run "$1") ) - if [[ $? != 0 ]]; then - unset COMPREPLY - elif [[ $SUPPRESS_SPACE == 1 ]] && [[ "$COMPREPLY" =~ [=/:]$ ]]; then - compopt -o nospace - fi -} -complete -o nospace -o default -o bashdefault -F _python_argcomplete query-exporter diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 1de6703..80b4b29 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -14,6 +14,8 @@ description: | - create or edit `/var/snap/query-exporter/current/config.yaml` with the configuration + - optionally, create a `/var/snap/query-exporter/current/.env` file with + environment variables definitions for additional config options - run `sudo snap restart query-exporter` Currently supported databases are: @@ -83,12 +85,16 @@ parts: - libodbc2 - libpq5 - libxml2 + override-build: | + craftctl default + _QUERY_EXPORTER_COMPLETE=bash_source query-exporter > "$CRAFT_PART_INSTALL/query-exporter.completion" prime: - bin/python3 - bin/query-exporter - etc - lib - pyvenv.cfg + - query-exporter.completion - usr/lib wrappers: From dccc38cc63e2d41e319e9edf4988ddeab9a4e718 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 16 Dec 2024 16:15:01 +0100 Subject: [PATCH 058/110] update doc --- CHANGES.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index d53ab22..7d3b171 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,31 @@ +v3.0.0 - XXXX-XX-XX +=================== + +- Convert all logging to structured (#199). +- Look for ``config.yaml`` configuration file by default, configuration file + can be optionally specified with ``--config``. +- Support passing configuration options via environment variables. +- Support loading ``.env`` file with environment variables for configuration. +- [docker] Run exporter from the ``/config`` directory, supporting having + ``.env`` file there. +- [snap] Run exporter from the ``$SNAP_DATA`` directory, supporting having + ``.env`` file there. + +**NOTE**: + This release introduces a few breaking changes from the 2.x series, + specificially: + + - The ``--log-level`` option now takes lowercase names for levels. + - The configuration file is no longer a required option, since + ``config.yaml`` in the current directory is looked up automatically. If a + different file is specified, it should be done as an optional parameter + with ``--config``. + - Metrics of type ``counter`` are now set by default to values returned by + queries. To preserve the old default behavior of incrementing it by the + returned value, ``increment`` should be set to ``true`` in the metric + configuration. + + v2.11.1 - 2024-11-19 ==================== From 0fcd90f1a80fb13d9f19e21f7c072417667dd4b6 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sun, 29 Dec 2024 23:49:00 +0100 Subject: [PATCH 059/110] add more typing to tests --- tests/conftest.py | 2 +- tests/loop_test.py | 162 +++++++++++++++++++++++++++++++-------------- 2 files changed, 113 insertions(+), 51 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5994bdd..ab0cbfc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ from query_exporter.db import DataBase, MetricResults, Query -__all__ = ["advance_time", "query_tracker"] +__all__ = ["QueryTracker", "advance_time", "query_tracker"] @pytest.fixture(autouse=True) diff --git a/tests/loop_test.py b/tests/loop_test.py index 6964d8b..31ca22f 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -1,6 +1,6 @@ import asyncio from collections import defaultdict -from collections.abc import Callable, Iterator +from collections.abc import AsyncIterator, Callable, Iterator from decimal import Decimal from pathlib import Path import typing as t @@ -8,15 +8,17 @@ from prometheus_aioexporter import MetricsRegistry import pytest +from pytest_mock import MockerFixture from pytest_structlog import StructuredLogCapture import yaml from query_exporter import loop -from query_exporter.config import ( - DataBaseConfig, - load_config, -) -from query_exporter.db import DataBase +from query_exporter.config import load_config +from query_exporter.db import DataBase, DataBaseConfig + +from .conftest import QueryTracker + +AdvanceTime = Callable[[float], t.Awaitable[None]] @pytest.fixture @@ -40,10 +42,13 @@ def registry() -> Iterator[MetricsRegistry]: yield MetricsRegistry() +MakeQueryLoop = Callable[[], loop.QueryLoop] + + @pytest.fixture async def make_query_loop( tmp_path: Path, config_data: dict[str, t.Any], registry: MetricsRegistry -) -> Iterator[Callable[[], MetricsRegistry]]: +) -> AsyncIterator[MakeQueryLoop]: query_loops = [] def make_loop() -> loop.QueryLoop: @@ -64,12 +69,15 @@ def make_loop() -> loop.QueryLoop: @pytest.fixture async def query_loop( - make_query_loop: Callable[[], loop.QueryLoop], -) -> Iterator[loop.QueryLoop]: + make_query_loop: MakeQueryLoop, +) -> AsyncIterator[loop.QueryLoop]: yield make_query_loop() -def metric_values(metric, by_labels=()): +MetricValues = list[int | float] | dict[tuple[str], list[int | float]] + + +def metric_values(metric, by_labels: tuple[str] = ()) -> MetricValues: """Return values for the metric.""" if metric._type == "gauge": suffix = "" @@ -145,7 +153,9 @@ def test_expire_no_labels(self) -> None: class TestQueryLoop: - async def test_start(self, query_tracker, query_loop) -> None: + async def test_start( + self, query_tracker: QueryTracker, query_loop + ) -> None: await query_loop.start() timed_call = query_loop._timed_calls["q"] assert timed_call.running @@ -158,7 +168,7 @@ async def test_stop(self, query_loop) -> None: assert not timed_call.running async def test_run_query( - self, query_tracker, query_loop, registry + self, query_tracker: QueryTracker, query_loop: loop.QueryLoop, registry ) -> None: await query_loop.start() await query_tracker.wait_results() @@ -173,16 +183,16 @@ async def test_run_query( async def test_run_scheduled_query( self, - mocker, - advance_time, - query_tracker, - registry, - config_data, - make_query_loop, + mocker: MockerFixture, + advance_time: AdvanceTime, + query_tracker: QueryTracker, + registry: MetricsRegistry, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: event_loop = asyncio.get_running_loop() - def croniter(*args): + def croniter(*args: t.Any) -> float: while True: # sync croniter time with the loop one yield event_loop.time() + 60 @@ -190,7 +200,7 @@ def croniter(*args): mock_croniter = mocker.patch.object(loop, "croniter") mock_croniter.side_effect = croniter # ensure that both clocks advance in sync - mocker.patch.object(loop.time, "time", lambda: event_loop.time()) + mocker.patch.object(loop.time, "time", lambda: event_loop.time()) # type: ignore del config_data["queries"]["q"]["interval"] config_data["queries"]["q"]["schedule"] = "*/2 * * * *" @@ -199,7 +209,11 @@ def croniter(*args): mock_croniter.assert_called_once() async def test_run_query_with_parameters( - self, query_tracker, registry, config_data, make_query_loop + self, + query_tracker: QueryTracker, + registry: MetricsRegistry, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: config_data["metrics"]["m"]["type"] = "counter" config_data["metrics"]["m"]["labels"] = ["l"] @@ -224,7 +238,11 @@ async def test_run_query_with_parameters( } async def test_run_query_null_value( - self, query_tracker, registry, config_data, make_query_loop + self, + query_tracker: QueryTracker, + registry: MetricsRegistry, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: config_data["queries"]["q"]["sql"] = "SELECT NULL AS m" query_loop = make_query_loop() @@ -242,10 +260,10 @@ async def test_run_query_null_value( ) async def test_run_query_counter( self, - query_tracker, - registry, - config_data, - make_query_loop, + query_tracker: QueryTracker, + registry: MetricsRegistry, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, increment: bool, value: float, ) -> None: @@ -264,7 +282,11 @@ async def test_run_query_counter( assert metric_values(metric) == [value] async def test_run_query_metrics_with_database_labels( - self, query_tracker, registry, config_data, make_query_loop + self, + query_tracker: QueryTracker, + registry: MetricsRegistry, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: config_data["databases"] = { "db1": {"dsn": "sqlite://", "labels": {"l1": "v1", "l2": "v2"}}, @@ -281,7 +303,7 @@ async def test_run_query_metrics_with_database_labels( } async def test_update_metric_decimal_value( - self, registry, make_query_loop + self, registry: MetricsRegistry, make_query_loop ) -> None: db = DataBase(DataBaseConfig(name="db", dsn="sqlite://")) query_loop = make_query_loop() @@ -294,8 +316,8 @@ async def test_update_metric_decimal_value( async def test_run_query_log( self, log: StructuredLogCapture, - query_tracker, - query_loop, + query_tracker: QueryTracker, + query_loop: loop.QueryLoop, ) -> None: await query_loop.start() await query_tracker.wait_queries() @@ -326,9 +348,9 @@ async def test_run_query_log( async def test_run_query_log_labels( self, log: StructuredLogCapture, - query_tracker, - config_data, - make_query_loop, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: config_data["metrics"]["m"]["labels"] = ["l"] config_data["queries"]["q"]["sql"] = 'SELECT 100.0 AS m, "foo" AS l' @@ -345,7 +367,11 @@ async def test_run_query_log_labels( ) async def test_run_query_increase_db_error_count( - self, query_tracker, config_data, make_query_loop, registry + self, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, + registry, ) -> None: config_data["databases"]["db"]["dsn"] = "sqlite:////invalid" query_loop = make_query_loop() @@ -355,7 +381,12 @@ async def test_run_query_increase_db_error_count( assert metric_values(queries_metric) == [1.0] async def test_run_query_increase_database_error_count( - self, mocker, query_tracker, config_data, make_query_loop, registry + self, + mocker, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, + registry, ) -> None: query_loop = make_query_loop() db = query_loop._databases["db"] @@ -367,7 +398,11 @@ async def test_run_query_increase_database_error_count( assert metric_values(queries_metric) == [1.0] async def test_run_query_increase_query_error_count( - self, query_tracker, config_data, make_query_loop, registry + self, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, + registry, ) -> None: config_data["queries"]["q"]["sql"] = "SELECT 100.0 AS a, 200.0 AS b" query_loop = make_query_loop() @@ -379,7 +414,11 @@ async def test_run_query_increase_query_error_count( } async def test_run_query_increase_timeout_count( - self, query_tracker, config_data, make_query_loop, registry + self, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, + registry, ) -> None: config_data["queries"]["q"]["timeout"] = 0.1 query_loop = make_query_loop() @@ -399,7 +438,10 @@ async def execute(sql, parameters): } async def test_run_query_at_interval( - self, advance_time, query_tracker, query_loop + self, + advance_time: AdvanceTime, + query_tracker: QueryTracker, + query_loop: loop.QueryLoop, ) -> None: await query_loop.start() await advance_time(0) # kick the first run @@ -413,7 +455,10 @@ async def test_run_query_at_interval( assert len(query_tracker.queries) == 2 async def test_run_timed_queries_invalid_result_count( - self, query_tracker, config_data, make_query_loop + self, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: config_data["queries"]["q"]["sql"] = "SELECT 100.0 AS a, 200.0 AS b" config_data["queries"]["q"]["interval"] = 1.0 @@ -432,7 +477,10 @@ async def test_run_timed_queries_invalid_result_count( assert len(query_tracker.results) == 0 async def test_run_timed_queries_invalid_result_count_stop_task( - self, query_tracker, config_data, make_query_loop + self, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: config_data["queries"]["q"]["sql"] = "SELECT 100.0 AS a, 200.0 AS b" config_data["queries"]["q"]["interval"] = 1.0 @@ -446,7 +494,11 @@ async def test_run_timed_queries_invalid_result_count_stop_task( assert query_loop._timed_calls == {} async def test_run_timed_queries_not_removed_if_not_failing_on_all_dbs( - self, tmp_path, query_tracker, config_data, make_query_loop + self, + tmp_path: Path, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: db1 = tmp_path / "db1.sqlite" db2 = tmp_path / "db2.sqlite" @@ -485,7 +537,10 @@ async def test_run_timed_queries_not_removed_if_not_failing_on_all_dbs( assert len(query_tracker.failures) == 1 async def test_run_aperiodic_queries( - self, query_tracker, config_data, make_query_loop + self, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: del config_data["queries"]["q"]["interval"] query_loop = make_query_loop() @@ -495,7 +550,10 @@ async def test_run_aperiodic_queries( assert len(query_tracker.queries) == 2 async def test_run_aperiodic_queries_invalid_result_count( - self, query_tracker, config_data, make_query_loop + self, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: config_data["queries"]["q"]["sql"] = "SELECT 100.0 AS a, 200.0 AS b" del config_data["queries"]["q"]["interval"] @@ -508,7 +566,11 @@ async def test_run_aperiodic_queries_invalid_result_count( assert len(query_tracker.queries) == 1 async def test_run_aperiodic_queries_not_removed_if_not_failing_on_all_dbs( - self, tmp_path, query_tracker, config_data, make_query_loop + self, + tmp_path: Path, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, ) -> None: db1 = tmp_path / "db1.sqlite" db2 = tmp_path / "db2.sqlite" @@ -547,12 +609,12 @@ async def test_run_aperiodic_queries_not_removed_if_not_failing_on_all_dbs( async def test_clear_expired_series( self, - tmp_path, - advance_time, - query_tracker, - config_data, - make_query_loop, - registry, + tmp_path: Path, + advance_time: AdvanceTime, + query_tracker: QueryTracker, + config_data: dict[str, t.Any], + make_query_loop: MakeQueryLoop, + registry: MetricsRegistry, ) -> None: db = tmp_path / "db.sqlite" config_data["databases"]["db"]["dsn"] = f"sqlite:///{db}" From c3ab60f4c1fb127c2a0fc04d38afd4ef83ba91da Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 30 Dec 2024 10:17:55 +0100 Subject: [PATCH 060/110] fix snap description --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 80b4b29..765d64d 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -12,7 +12,7 @@ description: | the command, managed via a Systemd service. To run the latter: - - create or edit `/var/snap/query-exporter/current/config.yaml` with the + - create `/var/snap/query-exporter/current/config.yaml` with the desired configuration - optionally, create a `/var/snap/query-exporter/current/.env` file with environment variables definitions for additional config options From 3fbe22acfb03a56edfa3934c15373ff77908fe1a Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 30 Dec 2024 10:18:43 +0100 Subject: [PATCH 061/110] Version 3.0.0 --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7d3b171..eb79868 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -v3.0.0 - XXXX-XX-XX +v3.0.0 - 2024-12-30 =================== - Convert all logging to structured (#199). From bfe5535512d41b27f8a93a1bb20337b7d99e2a47 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 30 Dec 2024 10:38:32 +0100 Subject: [PATCH 062/110] chore: fix typo in readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 43fff69..9621d10 100644 --- a/README.rst +++ b/README.rst @@ -567,7 +567,7 @@ where ``$CONFIG_DIR`` is the absolute path of a directory containing a ``config.yaml`` file, the configuration file to use. Alternatively, a volume name can be specified. -If, a ``.env`` file is present in the specified volume for ``/config``, its +If a ``.env`` file is present in the specified volume for ``/config``, its content is loaded and applied to the environment for the exporter. The location of the dotenv file can be customized by setting the ``QE_DOTENV`` environment variable. From 6b47645017f6f24c6715086521e60bab8030aae8 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 30 Dec 2024 10:38:37 +0100 Subject: [PATCH 063/110] chore(docker): build image on tag pushes --- .github/workflows/docker-image.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-image.yaml b/.github/workflows/docker-image.yaml index 291e9ee..fb0b27c 100644 --- a/.github/workflows/docker-image.yaml +++ b/.github/workflows/docker-image.yaml @@ -4,6 +4,8 @@ on: push: branches: - main + tags: + - "[0-9]+.[0-9]+.[0-9]+" pull_request: branches: - main From 966482b8799e396c5467cb619ea26a6fb4948fcc Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 30 Dec 2024 10:39:52 +0100 Subject: [PATCH 064/110] chore(snap): build images only for amd64 and arm64 --- snap/snapcraft.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 765d64d..1900c9f 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -40,8 +40,6 @@ base: core24 platforms: amd64: arm64: - s390x: - ppc64el: apps: daemon: From c2160925ec3ebdfdb6c9715b27aada20b2f66057 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 30 Dec 2024 10:51:01 +0100 Subject: [PATCH 065/110] chore(docker): document GHCR builds --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 9621d10..7d584ce 100644 --- a/README.rst +++ b/README.rst @@ -584,6 +584,11 @@ The image has support for connecting the following databases: A `Helm chart`_ to run the container in Kubernetes is also available. +Automated builds from the ``main`` branch are available on the `GitHub container registry`_ via:: + + docker pull ghcr.io/albertodonato/query-exporter:sha256-28058bd8c5acc97d57c1ad95f1a7395d9d43c30687459cd4adacc3e19d009996 + + ODBC driver version ~~~~~~~~~~~~~~~~~~~ @@ -604,6 +609,7 @@ by passing ``--build-arg ODBC_bVERSION_NUMBER``, e.g.:: .. _`Docker Hub`: https://hub.docker.com/r/adonato/query-exporter .. _`database-specific options`: databases.rst .. _`Helm chart`: https://github.com/makezbs/helm-charts/tree/main/charts/query-exporter +.. _`GitHub container registry`: https://github.com/albertodonato/query-exporter/pkgs/container/query-exporter .. |query-exporter logo| image:: https://raw.githubusercontent.com/albertodonato/query-exporter/main/logo.svg :alt: query-exporter logo From b6b18b397481c661611e205ee17c886cde762980 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 1 Jan 2025 16:06:21 +0100 Subject: [PATCH 066/110] chore: split configuration file format docs (#215) --- README.rst | 389 +--------------------------- docs/configuration.rst | 268 +++++++++++++++++++ databases.rst => docs/databases.rst | 0 3 files changed, 275 insertions(+), 382 deletions(-) create mode 100644 docs/configuration.rst rename databases.rst => docs/databases.rst (100%) diff --git a/README.rst b/README.rst index 7d584ce..6cf3cb7 100644 --- a/README.rst +++ b/README.rst @@ -23,10 +23,6 @@ to perform to update metrics. The configuration file can be overridden by passing the ``--config`` option, or setting the ``QE_CONFIG`` environment variable. - -Configuration file format -------------------------- - A sample configuration file for the application looks like this: .. code:: yaml @@ -94,263 +90,11 @@ A sample configuration file for the application looks like this: ORDER BY random() LIMIT 1 -``databases`` section -~~~~~~~~~~~~~~~~~~~~~ - -This section contains definitions for databases to connect to. Key names are -arbitrary and only used to reference databases in the ``queries`` section. - -Each database definitions can have the following keys: - -``dsn``: - database connection details. - - It can be provided as a string in the following format:: - - dialect[+driver]://[username:password][@host:port]/database[?option=value&...] - - (see `SQLAlchemy documentation`_ for details on available engines and - options), or as key/value pairs: - - .. code:: yaml - - dialect: [+driver] - user: - password: - host: - port: - database: - options: - : - : - - All entries are optional, except ``dialect``. - - **Note**: in the string form, username, password and options need to be - URL-encoded, whereas this is done automatically for the key/value form. - - See `database-specific options`_ page for some extra details on database - configuration options. - - It's also possible to get the connection string indirectly from other sources: - - - from an environment variable (e.g. ``$CONNECTION_STRING``) by setting ``dsn`` to:: - - env:CONNECTION_STRING - - - from a file, containing only the DSN value, by setting ``dsn`` to:: - - file:/path/to/file - - These forms only support specifying the actual DNS in the string form. - -``connect-sql``: - An optional list of queries to run right after database connection. This can - be used to set up connection-wise parameters and configurations. - -``keep-connected``: - whether to keep the connection open for the database between queries, or - disconnect after each one. If not specified, defaults to ``true``. Setting - this option to ``false`` might be useful if queries on a database are run - with very long interval, to avoid holding idle connections. - -``autocommit``: - whether to set autocommit for the database connection. If not specified, - defaults to ``true``. This should only be changed to ``false`` if specific - queries require it. - -``labels``: - an optional mapping of label names and values to tag metrics collected from each database. - When labels are used, all databases must define the same set of labels. - -``metrics`` section -~~~~~~~~~~~~~~~~~~~ - -This section contains Prometheus_ metrics definitions. Keys are used as metric -names, and must therefore be valid metric identifiers. - -Each metric definition can have the following keys: - -``type``: - the type of the metric, must be specified. The following metric types are - supported: - - - ``counter``: value is incremented with each result from queries - - ``enum``: value is set with each result from queries - - ``gauge``: value is set with each result from queries - - ``histogram``: each result from queries is added to observations - - ``summary``: each result from queries is added to observations - -``description``: - an optional description of the metric. - -``labels``: - an optional list of label names to apply to the metric. - - If specified, queries updating the metric must return rows that include - values for each label in addition to the metric value. Column names must - match metric and labels names. - -``buckets``: - for ``histogram`` metrics, a list of buckets for the metrics. - - If not specified, default buckets are applied. - -``states``: - for ``enum`` metrics, a list of string values for possible states. - - Queries for updating the enum must return valid states. - -``expiration``: - the amount of time after which a series for the metric is cleared if no new - value is collected. - - Last report times are tracked independently for each set of label values for - the metric. - - This can be useful for metric series that only last for a certain amount of - time, to avoid an ever-increasing collection of series. - - The value is interpreted as seconds if no suffix is specified; valid suffixes - are ``s``, ``m``, ``h``, ``d``. Only integer values are accepted. - -``increment``: - for ``counter`` metrics, whether to increment the value by the query result, - or set the value to it. - - By default, counters are set to the value returned by the query. If this is - set to ``true``, instead, the metric value will be incremented by the result - of the query. - - -``queries`` section -~~~~~~~~~~~~~~~~~~~ - -This section contains definitions for queries to perform. Key names are -arbitrary and only used to identify queries in logs. - -Each query definition can have the following keys: - -``databases``: - the list of databases to run the query on. - - Names must match those defined in the ``databases`` section. - - Metrics are automatically tagged with the ``database`` label so that - independent series are generated for each database that a query is run on. - -``interval``: - the time interval at which the query is run. - - The value is interpreted as seconds if no suffix is specified; valid suffixes - are ``s``, ``m``, ``h``, ``d``. Only integer values are accepted. - - If a value is specified for ``interval``, a ``schedule`` can't be specified. - - If no value is specified (or specified as ``null``), the query is only - executed upon HTTP requests. - -``metrics``: - the list of metrics that the query updates. - - Names must match those defined in the ``metrics`` section. - -``parameters``: - an optional list or dictionary of parameters sets to run the query with. - - If specified as a list, the query will be run once for every set of - parameters specified in this list, for every interval. - - Each parameter set must be a dictionary where keys must match parameters - names from the query SQL (e.g. ``:param``). - - As an example: - - .. code:: yaml - - query: - databases: [db] - metrics: [metric] - sql: | - SELECT COUNT(*) AS metric FROM table - WHERE id > :param1 AND id < :param2 - parameters: - - param1: 10 - param2: 20 - - param1: 30 - param2: 40 - - If specified as a dictionary, it's used as a multidimensional matrix of - parameters lists to run the query with. - The query will be run once for each permutation of parameters. - - If a query is specified with parameters as matrix in its ``sql``, it will be run once - for every permutation in matrix of parameters, for every interval. - - Variable format in sql query: ``:{top_level_key}__{inner_key}`` - - .. code:: yaml - - query: - databases: [db] - metrics: [apps_count] - sql: | - SELECT COUNT(1) AS apps_count FROM apps_list - WHERE os = :os__name AND arch = :os__arch AND lang = :lang__name - parameters: - os: - - name: MacOS - arch: arm64 - - name: Linux - arch: amd64 - - name: Windows - arch: amd64 - lang: - - name: Python3 - - name: Java - - name: TypeScript - - This example will generate 9 queries with all permutations of ``os`` and - ``lang`` parameters. - -``schedule``: - a schedule for executing queries at specific times. - - This is expressed as a Cron-like format string (e.g. ``*/5 * * * *`` to run - every five minutes). - - If a value is specified for ``schedule``, an ``interval`` can't be specified. - - If no value is specified (or specified as ``null``), the query is only - executed upon HTTP requests. - -``sql``: - the SQL text of the query. - - The query must return columns with names that match those of the metrics - defined in ``metrics``, plus those of labels (if any) for all these metrics. - - .. code:: yaml - - query: - databases: [db] - metrics: [metric1, metric2] - sql: SELECT 10.0 AS metric1, 20.0 AS metric2 - - will update ``metric1`` to ``10.0`` and ``metric2`` to ``20.0``. - - **Note**: - since ``:`` is used for parameter markers (see ``parameters`` above), - literal single ``:`` at the beginning of a word must be escaped with - backslash (e.g. ``SELECT '\:bar' FROM table``). There's no need to escape - when the colon occurs inside a word (e.g. ``SELECT 'foo:bar' FROM table``). - -``timeout``: - a value in seconds after which the query is timed out. - - If specified, it must be a multiple of 0.1. +See the `configuration file format`_ documentation for complete details on +availble configuration options. + Metrics endpoint ---------------- @@ -360,125 +104,8 @@ endpoint. By default, the port is bound on ``localhost``. Note that if the name resolves both IPv4 and IPv6 addressses, the exporter will bind on both. -For the configuration above, the endpoint would return something like this:: - - # HELP database_errors_total Number of database errors - # TYPE database_errors_total counter - # HELP queries_total Number of database queries - # TYPE queries_total counter - queries_total{app="app1",database="db1",query="query1",region="us1",status="success"} 50.0 - queries_total{app="app1",database="db2",query="query2",region="us2",status="success"} 13.0 - queries_total{app="app1",database="db1",query="query2",region="us1",status="success"} 13.0 - queries_total{app="app1",database="db2",query="query3",region="us2",status="error"} 1.0 - # HELP queries_created Number of database queries - # TYPE queries_created gauge - queries_created{app="app1",database="db1",query="query1",region="us1",status="success"} 1.5945442444463024e+09 - queries_created{app="app1",database="db2",query="query2",region="us2",status="success"} 1.5945442444471517e+09 - queries_created{app="app1",database="db1",query="query2",region="us1",status="success"} 1.5945442444477117e+09 - queries_created{app="app1",database="db2",query="query3",region="us2",status="error"} 1.5945444000140696e+09 - # HELP query_latency Query execution latency - # TYPE query_latency histogram - query_latency_bucket{app="app1",database="db1",le="0.005",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="0.01",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="0.025",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="0.05",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="0.075",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="0.1",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="0.25",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="0.5",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="0.75",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="1.0",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="2.5",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="5.0",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="7.5",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="10.0",query="query1",region="us1"} 50.0 - query_latency_bucket{app="app1",database="db1",le="+Inf",query="query1",region="us1"} 50.0 - query_latency_count{app="app1",database="db1",query="query1",region="us1"} 50.0 - query_latency_sum{app="app1",database="db1",query="query1",region="us1"} 0.004666365042794496 - query_latency_bucket{app="app1",database="db2",le="0.005",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="0.01",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="0.025",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="0.05",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="0.075",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="0.1",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="0.25",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="0.5",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="0.75",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="1.0",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="2.5",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="5.0",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="7.5",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="10.0",query="query2",region="us2"} 13.0 - query_latency_bucket{app="app1",database="db2",le="+Inf",query="query2",region="us2"} 13.0 - query_latency_count{app="app1",database="db2",query="query2",region="us2"} 13.0 - query_latency_sum{app="app1",database="db2",query="query2",region="us2"} 0.012369773990940303 - query_latency_bucket{app="app1",database="db1",le="0.005",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="0.01",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="0.025",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="0.05",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="0.075",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="0.1",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="0.25",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="0.5",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="0.75",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="1.0",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="2.5",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="5.0",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="7.5",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="10.0",query="query2",region="us1"} 13.0 - query_latency_bucket{app="app1",database="db1",le="+Inf",query="query2",region="us1"} 13.0 - query_latency_count{app="app1",database="db1",query="query2",region="us1"} 13.0 - query_latency_sum{app="app1",database="db1",query="query2",region="us1"} 0.004745393933262676 - # HELP query_latency_created Query execution latency - # TYPE query_latency_created gauge - query_latency_created{app="app1",database="db1",query="query1",region="us1"} 1.594544244446163e+09 - query_latency_created{app="app1",database="db2",query="query2",region="us2"} 1.5945442444470239e+09 - query_latency_created{app="app1",database="db1",query="query2",region="us1"} 1.594544244447551e+09 - # HELP query_timestamp Query last execution timestamp - # TYPE query_timestamp gauge - query_timestamp{app="app1",database="db2",query="query2",region="us2"} 1.594544244446199e+09 - query_timestamp{app="app1",database="db1",query="query1",region="us1"} 1.594544244452181e+09 - query_timestamp{app="app1",database="db1",query="query2",region="us1"} 1.594544244481839e+09 - # HELP metric1 A sample gauge - # TYPE metric1 gauge - metric1{app="app1",database="db1",region="us1"} -3561.0 - # HELP metric2 A sample summary - # TYPE metric2 summary - metric2_count{app="app1",database="db2",l1="value1",l2="value2",region="us2"} 13.0 - metric2_sum{app="app1",database="db2",l1="value1",l2="value2",region="us2"} 58504.0 - metric2_count{app="app1",database="db1",l1="value1",l2="value2",region="us1"} 13.0 - metric2_sum{app="app1",database="db1",l1="value1",l2="value2",region="us1"} 75262.0 - # HELP metric2_created A sample summary - # TYPE metric2_created gauge - metric2_created{app="app1",database="db2",l1="value1",l2="value2",region="us2"} 1.594544244446819e+09 - metric2_created{app="app1",database="db1",l1="value1",l2="value2",region="us1"} 1.594544244447339e+09 - # HELP metric3 A sample histogram - # TYPE metric3 histogram - metric3_bucket{app="app1",database="db2",le="10.0",region="us2"} 1.0 - metric3_bucket{app="app1",database="db2",le="20.0",region="us2"} 1.0 - metric3_bucket{app="app1",database="db2",le="50.0",region="us2"} 2.0 - metric3_bucket{app="app1",database="db2",le="100.0",region="us2"} 3.0 - metric3_bucket{app="app1",database="db2",le="1000.0",region="us2"} 13.0 - metric3_bucket{app="app1",database="db2",le="+Inf",region="us2"} 13.0 - metric3_count{app="app1",database="db2",region="us2"} 13.0 - metric3_sum{app="app1",database="db2",region="us2"} 5016.0 - metric3_bucket{app="app1",database="db1",le="10.0",region="us1"} 0.0 - metric3_bucket{app="app1",database="db1",le="20.0",region="us1"} 0.0 - metric3_bucket{app="app1",database="db1",le="50.0",region="us1"} 0.0 - metric3_bucket{app="app1",database="db1",le="100.0",region="us1"} 0.0 - metric3_bucket{app="app1",database="db1",le="1000.0",region="us1"} 13.0 - metric3_bucket{app="app1",database="db1",le="+Inf",region="us1"} 13.0 - metric3_count{app="app1",database="db1",region="us1"} 13.0 - metric3_sum{app="app1",database="db1",region="us1"} 5358.0 - # HELP metric3_created A sample histogram - # TYPE metric3_created gauge - metric3_created{app="app1",database="db2",region="us2"} 1.5945442444469101e+09 - metric3_created{app="app1",database="db1",region="us1"} 1.5945442444474254e+09 - # HELP metric4 A sample enum - # TYPE metric4 gauge - metric4{app="app1",database="db2",metric4="foo",region="us2"} 0.0 - metric4{app="app1",database="db2",metric4="bar",region="us2"} 0.0 - metric4{app="app1",database="db2",metric4="baz",region="us2"} 1.0 +Both port and host can be overridden via command-line parameters and +environment variables. Builtin metrics @@ -505,7 +132,7 @@ included by passing ``--process-stats`` in the command line. Debugging / Logs ---------------- -You can enable extended logging using the ``-L`` (or ``-log-level``) command +You can enable extended logging using the ``-L`` (or ``--log-level``) command line switch. Possible log levels are ``critical``, ``error``, ``warning``, ``info``, ``debug``. @@ -600,14 +227,12 @@ by passing ``--build-arg ODBC_bVERSION_NUMBER``, e.g.:: .. _Prometheus: https://prometheus.io/ .. _SQLAlchemy: https://www.sqlalchemy.org/ -.. _`SQLAlchemy documentation`: - http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls .. _`supported databases`: http://docs.sqlalchemy.org/en/latest/core/engines.html#supported-databases .. _`Snap Store`: https://snapcraft.io .. _Docker: http://docker.com/ .. _`Docker Hub`: https://hub.docker.com/r/adonato/query-exporter -.. _`database-specific options`: databases.rst +.. _`configuration file format`: docs/configuration.rst .. _`Helm chart`: https://github.com/makezbs/helm-charts/tree/main/charts/query-exporter .. _`GitHub container registry`: https://github.com/albertodonato/query-exporter/pkgs/container/query-exporter diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 0000000..2365481 --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,268 @@ +Configuration file format +========================= + +Configuration is provided as a YAML file, composed by a few sections, as +described in the following sections. + + +``databases`` section +--------------------- + +This section contains definitions for databases to connect to. Key names are +arbitrary and only used to reference databases in the ``queries`` section. + +Each database definitions can have the following keys: + +``dsn``: + database connection details. + + It can be provided as a string in the following format:: + + dialect[+driver]://[username:password][@host:port]/database[?option=value&...] + + or as a map with the following keys: + + .. code:: yaml + + dialect: [+driver] + user: + password: + host: + port: + database: + options: + : + : + + All entries are optional, except ``dialect``. + + See `SQLAlchemy documentation`_ for details on available engines and options + and the `database-specific options`_ page for some extra details on database + configuration options. + + **Note**: in the string form, username, password and options need to be + URL-encoded, whereas this is done automatically for the key/value form. + + It's also possible to get the connection string indirectly from other sources: + + - from an environment variable (e.g. ``$CONNECTION_STRING``) by setting ``dsn`` to:: + + env:CONNECTION_STRING + + - from a file, containing only the DSN value, by setting ``dsn`` to:: + + file:/path/to/file + + These forms only support specifying the actual DNS in the string form. + +``connect-sql``: + An optional list of queries to run right after database connection. This can + be used to set up connection-wise parameters and configurations. + +``keep-connected``: + whether to keep the connection open for the database between queries, or + disconnect after each one. If not specified, defaults to ``true``. Setting + this option to ``false`` might be useful if queries on a database are run + with very long interval, to avoid holding idle connections. + +``autocommit``: + whether to set autocommit for the database connection. If not specified, + defaults to ``true``. This should only be changed to ``false`` if specific + queries require it. + +``labels``: + an optional mapping of label names and values to tag metrics collected from each database. + When labels are used, all databases must define the same set of labels. + + +``metrics`` section +------------------- + +This section contains Prometheus metrics definitions. Keys are used as metric +names, and must therefore be valid metric identifiers. + +Each metric definition can have the following keys: + +``type``: + the type of the metric, must be specified. The following metric types are + supported: + + - ``counter``: value is incremented with each result from queries + - ``enum``: value is set with each result from queries + - ``gauge``: value is set with each result from queries + - ``histogram``: each result from queries is added to observations + - ``summary``: each result from queries is added to observations + +``description``: + an optional description of the metric. + +``labels``: + an optional list of label names to apply to the metric. + + If specified, queries updating the metric must return rows that include + values for each label in addition to the metric value. Column names must + match metric and labels names. + +``buckets``: + for ``histogram`` metrics, a list of buckets for the metrics. + + If not specified, default buckets are applied. + +``states``: + for ``enum`` metrics, a list of string values for possible states. + + Queries for updating the enum must return valid states. + +``expiration``: + the amount of time after which a series for the metric is cleared if no new + value is collected. + + Last report times are tracked independently for each set of label values for + the metric. + + This can be useful for metric series that only last for a certain amount of + time, to avoid an ever-increasing collection of series. + + The value is interpreted as seconds if no suffix is specified; valid suffixes + are ``s``, ``m``, ``h``, ``d``. Only integer values are accepted. + +``increment``: + for ``counter`` metrics, whether to increment the value by the query result, + or set the value to it. + + By default, counters are set to the value returned by the query. If this is + set to ``true``, instead, the metric value will be incremented by the result + of the query. + + +``queries`` section +------------------- + +This section contains definitions for queries to perform. Key names are +arbitrary and only used to identify queries in logs. + +Each query definition can have the following keys: + +``databases``: + the list of databases to run the query on. + + Names must match those defined in the ``databases`` section. + + Metrics are automatically tagged with the ``database`` label so that + independent series are generated for each database that a query is run on. + +``interval``: + the time interval at which the query is run. + + The value is interpreted as seconds if no suffix is specified; valid suffixes + are ``s``, ``m``, ``h``, ``d``. Only integer values are accepted. + + If a value is specified for ``interval``, a ``schedule`` can't be specified. + + If no value is specified (or specified as ``null``), the query is only + executed upon HTTP requests. + +``metrics``: + the list of metrics that the query updates. + + Names must match those defined in the ``metrics`` section. + +``parameters``: + an optional list or dictionary of parameters sets to run the query with. + + If specified as a list, the query will be run once for every set of + parameters specified in this list, for every interval. + + Each parameter set must be a dictionary where keys must match parameters + names from the query SQL (e.g. ``:param``). + + As an example: + + .. code:: yaml + + query: + databases: [db] + metrics: [metric] + sql: | + SELECT COUNT(*) AS metric FROM table + WHERE id > :param1 AND id < :param2 + parameters: + - param1: 10 + param2: 20 + - param1: 30 + param2: 40 + + If specified as a dictionary, it's used as a multidimensional matrix of + parameters lists to run the query with. + The query will be run once for each permutation of parameters. + + If a query is specified with parameters as matrix in its ``sql``, it will be run once + for every permutation in matrix of parameters, for every interval. + + Variable format in sql query: ``:{top_level_key}__{inner_key}`` + + .. code:: yaml + + query: + databases: [db] + metrics: [apps_count] + sql: | + SELECT COUNT(1) AS apps_count FROM apps_list + WHERE os = :os__name AND arch = :os__arch AND lang = :lang__name + parameters: + os: + - name: MacOS + arch: arm64 + - name: Linux + arch: amd64 + - name: Windows + arch: amd64 + lang: + - name: Python3 + - name: Java + - name: TypeScript + + This example will generate 9 queries with all permutations of ``os`` and + ``lang`` parameters. + +``schedule``: + a schedule for executing queries at specific times. + + This is expressed as a Cron-like format string (e.g. ``*/5 * * * *`` to run + every five minutes). + + If a value is specified for ``schedule``, an ``interval`` can't be specified. + + If no value is specified (or specified as ``null``), the query is only + executed upon HTTP requests. + +``sql``: + the SQL text of the query. + + The query must return columns with names that match those of the metrics + defined in ``metrics``, plus those of labels (if any) for all these metrics. + + .. code:: yaml + + query: + databases: [db] + metrics: [metric1, metric2] + sql: SELECT 10.0 AS metric1, 20.0 AS metric2 + + will update ``metric1`` to ``10.0`` and ``metric2`` to ``20.0``. + + **Note**: + since ``:`` is used for parameter markers (see ``parameters`` above), + literal single ``:`` at the beginning of a word must be escaped with + backslash (e.g. ``SELECT '\:bar' FROM table``). There's no need to escape + when the colon occurs inside a word (e.g. ``SELECT 'foo:bar' FROM table``). + +``timeout``: + a value in seconds after which the query is timed out. + + If specified, it must be a multiple of 0.1. + + +.. _`database-specific options`: databases.rst +.. _`SQLAlchemy documentation`: + http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls diff --git a/databases.rst b/docs/databases.rst similarity index 100% rename from databases.rst rename to docs/databases.rst From 4a859f9c2f0f11cc4972e1176b0be7999d121eac Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 2 Jan 2025 10:21:11 +0100 Subject: [PATCH 067/110] feat: support YAML tags for env, file, and include (fixes #188) (#216) --- docs/configuration.rst | 37 ++++++++++----- examples/oracle-stats.yaml | 2 +- examples/postgresql-stats.yaml | 2 +- query_exporter/config.py | 13 ++++- query_exporter/yaml.py | 64 +++++++++++++++++++++++++ tests/config_test.py | 18 ++++++- tests/yaml_test.py | 87 ++++++++++++++++++++++++++++++++++ 7 files changed, 207 insertions(+), 16 deletions(-) create mode 100644 query_exporter/yaml.py create mode 100644 tests/yaml_test.py diff --git a/docs/configuration.rst b/docs/configuration.rst index 2365481..dd26d18 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -4,6 +4,29 @@ Configuration file format Configuration is provided as a YAML file, composed by a few sections, as described in the following sections. +The following tags are supported in the configuration file: + +``!include ``: + include the content of another YAML file. This allows modularizing + configuration. + + If the specified path is not absolute, it's considered relative to the + including file. + +``!file ``: + include the text content of a file as a string. + + If the specified path is not absolute, it's considered relative to the + including file. + +``!env ``: + expand to the value of the specified environment variable. + + Note that the value of the variable is interpreted as YAML (and thus JSON), + allowing for specifying values other than strings (e.g. integers/floats). + + The specified variable must be set. + ``databases`` section --------------------- @@ -43,17 +66,9 @@ Each database definitions can have the following keys: **Note**: in the string form, username, password and options need to be URL-encoded, whereas this is done automatically for the key/value form. - It's also possible to get the connection string indirectly from other sources: - - - from an environment variable (e.g. ``$CONNECTION_STRING``) by setting ``dsn`` to:: - - env:CONNECTION_STRING - - - from a file, containing only the DSN value, by setting ``dsn`` to:: - - file:/path/to/file - - These forms only support specifying the actual DNS in the string form. + **Note**: use of the ``env:`` and ``file:`` prefixes in the string form is + deprecated, and will be dropped in the 4.0 release. Use ``!env`` and + ``!file`` YAML tags instead. ``connect-sql``: An optional list of queries to run right after database connection. This can diff --git a/examples/oracle-stats.yaml b/examples/oracle-stats.yaml index d8ec507..94fab89 100644 --- a/examples/oracle-stats.yaml +++ b/examples/oracle-stats.yaml @@ -7,7 +7,7 @@ databases: oracle: - dsn: env:ORACLE_DATABASE_DSN + dsn: !env ORACLE_DATABASE_DSN metrics: oracle_sessions: diff --git a/examples/postgresql-stats.yaml b/examples/postgresql-stats.yaml index 3e48c8b..7574ffd 100644 --- a/examples/postgresql-stats.yaml +++ b/examples/postgresql-stats.yaml @@ -11,7 +11,7 @@ databases: pg: - dsn: env:PG_DATABASE_DSN + dsn: !env PG_DATABASE_DSN metrics: pg_process: diff --git a/query_exporter/config.py b/query_exporter/config.py index ed80880..7687fba 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -28,6 +28,7 @@ Query, QueryMetric, ) +from .yaml import load_yaml_config # metric for counting database errors DB_ERRORS_METRIC_NAME = "database_errors" @@ -105,8 +106,10 @@ def load_config( if logger is None: logger = structlog.get_logger() - with config_path.open() as fd: - data = defaultdict(dict, yaml.safe_load(fd)) + try: + data = defaultdict(dict, load_yaml_config(config_path)) + except yaml.scanner.ScannerError as e: + raise ConfigError(str(e)) _validate_config(data) databases, database_labels = _get_databases(data["databases"], env) extra_labels = frozenset([DATABASE_LABEL]) | database_labels @@ -347,6 +350,12 @@ def from_file(filename: str) -> str: source, value = dsn.split(":", 1) handler = origins.get(source) if handler is not None: + logger = structlog.get_logger() + logger.warn( + f"deprecated DSN source '{dsn}', use '!{source} {value}' instead", + source=source, + value=value, + ) return handler(value) return dsn diff --git a/query_exporter/yaml.py b/query_exporter/yaml.py new file mode 100644 index 0000000..ab3cad9 --- /dev/null +++ b/query_exporter/yaml.py @@ -0,0 +1,64 @@ +import os +from pathlib import Path +import typing as t + +import yaml + + +def load_yaml_config(path: Path) -> t.Any: + """Load a YAML document from a file.""" + + class ConfigLoader(yaml.SafeLoader): + """Subclass supporting tags.""" + + base_path: t.ClassVar[Path] + + def config_loader(path: Path) -> type[ConfigLoader]: + class ConfigLoaderWithPath(ConfigLoader): + base_path = path + + return ConfigLoaderWithPath + + def tag_env(loader: ConfigLoader, node: yaml.nodes.ScalarNode) -> t.Any: + env = loader.construct_scalar(node) + value = os.getenv(env) + if value is None: + raise yaml.scanner.ScannerError( + "while processing 'env' tag", + None, + f"variable {env} undefined", + loader.get_mark(), # type: ignore + ) + return yaml.safe_load(value) + + def tag_file(loader: ConfigLoader, node: yaml.nodes.ScalarNode) -> str: + path = loader.base_path / loader.construct_scalar(node) + if not path.is_file(): + raise yaml.scanner.ScannerError( + "while processing 'file' tag", + None, + f"file {path} not found", + loader.get_mark(), # type: ignore + ) + return path.read_text().strip() + + def tag_include( + loader: ConfigLoader, node: yaml.nodes.ScalarNode + ) -> t.Any: + path = loader.base_path / loader.construct_scalar(node) + if not path.is_file(): + raise yaml.scanner.ScannerError( + "while processing 'include' tag", + None, + f"file {path} not found", + loader.get_mark(), # type: ignore + ) + with path.open() as fd: + return yaml.load(fd, config_loader(path.parent)) + + ConfigLoader.add_constructor("!env", tag_env) + ConfigLoader.add_constructor("!file", tag_file) + ConfigLoader.add_constructor("!include", tag_include) + + with path.open() as fd: + return yaml.load(fd, config_loader(path.parent)) diff --git a/tests/config_test.py b/tests/config_test.py index 2e97570..e2d9085 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -128,6 +128,13 @@ def write(data: dict[str, t.Any]) -> Path: class TestLoadConfig: + def test_load_invalid_format(self, tmp_path: Path) -> None: + config_file = tmp_path / "config" + config_file.write_text("foo: !env UNSET") + with pytest.raises(ConfigError) as err: + load_config(config_file) + assert "variable UNSET undefined" in str(err.value) + def test_load_databases_section(self, write_config: ConfigWriter) -> None: cfg = { "databases": { @@ -156,7 +163,9 @@ def test_load_databases_section(self, write_config: ConfigWriter) -> None: assert not database2.autocommit def test_load_databases_dsn_from_env( - self, write_config: ConfigWriter + self, + log: StructuredLogCapture, + write_config: ConfigWriter, ) -> None: cfg = { "databases": {"db1": {"dsn": "env:FOO"}}, @@ -166,6 +175,9 @@ def test_load_databases_dsn_from_env( config_file = write_config(cfg) config = load_config(config_file, env={"FOO": "sqlite://"}) assert config.databases["db1"].dsn == "sqlite://" + assert log.has( + "deprecated DSN source 'env:FOO', use '!env FOO' instead" + ) def test_load_databases_missing_dsn( self, write_config: ConfigWriter @@ -262,6 +274,7 @@ def test_load_databases_dsn_undefined_env( def test_load_databases_dsn_from_file( self, tmp_path: Path, + log: StructuredLogCapture, write_config: ConfigWriter, ) -> None: dsn = "sqlite:///foo" @@ -275,6 +288,9 @@ def test_load_databases_dsn_from_file( config_file = write_config(cfg) config = load_config(config_file) assert config.databases["db1"].dsn == dsn + assert log.has( + f"deprecated DSN source 'file:{dsn_path}', use '!file {dsn_path}' instead" + ) def test_load_databases_dsn_from_file_not_found( self, write_config: ConfigWriter diff --git a/tests/yaml_test.py b/tests/yaml_test.py new file mode 100644 index 0000000..0560f65 --- /dev/null +++ b/tests/yaml_test.py @@ -0,0 +1,87 @@ +from pathlib import Path +from textwrap import dedent +import typing as t + +import pytest +import yaml + +from query_exporter.yaml import load_yaml_config + + +class TestLoadYAMLConfig: + def test_load(self, tmp_path: Path) -> None: + config = tmp_path / "config.yaml" + config.write_text( + dedent( + """ + a: b + c: d + """ + ) + ) + assert load_yaml_config(config) == {"a": "b", "c": "d"} + + @pytest.mark.parametrize("env_value", ["foo", 3, False, {"foo": "bar"}]) + def test_load_env( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, env_value: t.Any + ) -> None: + monkeypatch.setenv("FOO", yaml.dump(env_value)) + config = tmp_path / "config.yaml" + config.write_text("x: !env FOO") + assert load_yaml_config(config) == {"x": env_value} + + def test_load_env_not_found(self, tmp_path: Path) -> None: + config = tmp_path / "config.yaml" + config.write_text("x: !env FOO") + with pytest.raises(yaml.scanner.ScannerError) as err: + load_yaml_config(config) + assert "variable FOO undefined" in str(err.value) + + def test_load_file_relative_path(self, tmp_path: Path) -> None: + (tmp_path / "foo.txt").write_text("some text") + config = tmp_path / "config.yaml" + config.write_text("x: !file foo.txt") + assert load_yaml_config(config) == {"x": "some text"} + + def test_load_file_absolute_path(self, tmp_path: Path) -> None: + text_file = tmp_path / "foo.txt" + text_file.write_text("some text") + config = tmp_path / "config.yaml" + config.write_text(f"x: !file {text_file.absolute()!s}") + assert load_yaml_config(config) == {"x": "some text"} + + def test_load_file_not_found(self, tmp_path: Path) -> None: + config = tmp_path / "config.yaml" + config.write_text("x: !file not-here.txt") + with pytest.raises(yaml.scanner.ScannerError) as err: + load_yaml_config(config) + assert f"file {tmp_path / 'not-here.txt'} not found" in str(err.value) + + def test_load_include_relative_path(self, tmp_path: Path) -> None: + (tmp_path / "foo.yaml").write_text("foo: bar") + config = tmp_path / "config.yaml" + config.write_text("x: !include foo.yaml") + assert load_yaml_config(config) == {"x": {"foo": "bar"}} + + def test_load_include_absolute_path(self, tmp_path: Path) -> None: + other_file = tmp_path / "foo.yaml" + other_file.write_text("foo: bar") + config = tmp_path / "config.yaml" + config.write_text(f"x: !include {other_file.absolute()!s}") + assert load_yaml_config(config) == {"x": {"foo": "bar"}} + + def test_load_include_multiple(self, tmp_path: Path) -> None: + subdir = tmp_path / "subdir" + subdir.mkdir() + (subdir / "bar.yaml").write_text("[a, b, c]") + (subdir / "foo.yaml").write_text("foo: !include bar.yaml") + config = tmp_path / "config.yaml" + config.write_text("x: !include subdir/foo.yaml") + assert load_yaml_config(config) == {"x": {"foo": ["a", "b", "c"]}} + + def test_load_include_not_found(self, tmp_path: Path) -> None: + config = tmp_path / "config.yaml" + config.write_text("x: !include not-here.yaml") + with pytest.raises(yaml.scanner.ScannerError) as err: + load_yaml_config(config) + assert f"file {tmp_path / 'not-here.yaml'} not found" in str(err.value) From f394f392f814e7d517d43a792b77056ef8dd6a0e Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 2 Jan 2025 10:42:54 +0100 Subject: [PATCH 068/110] chore: split yaml module code --- query_exporter/yaml.py | 108 +++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/query_exporter/yaml.py b/query_exporter/yaml.py index ab3cad9..222c200 100644 --- a/query_exporter/yaml.py +++ b/query_exporter/yaml.py @@ -8,57 +8,61 @@ def load_yaml_config(path: Path) -> t.Any: """Load a YAML document from a file.""" - class ConfigLoader(yaml.SafeLoader): - """Subclass supporting tags.""" - - base_path: t.ClassVar[Path] - - def config_loader(path: Path) -> type[ConfigLoader]: - class ConfigLoaderWithPath(ConfigLoader): - base_path = path - - return ConfigLoaderWithPath - - def tag_env(loader: ConfigLoader, node: yaml.nodes.ScalarNode) -> t.Any: - env = loader.construct_scalar(node) - value = os.getenv(env) - if value is None: - raise yaml.scanner.ScannerError( - "while processing 'env' tag", - None, - f"variable {env} undefined", - loader.get_mark(), # type: ignore - ) - return yaml.safe_load(value) - - def tag_file(loader: ConfigLoader, node: yaml.nodes.ScalarNode) -> str: - path = loader.base_path / loader.construct_scalar(node) - if not path.is_file(): - raise yaml.scanner.ScannerError( - "while processing 'file' tag", - None, - f"file {path} not found", - loader.get_mark(), # type: ignore - ) - return path.read_text().strip() - - def tag_include( - loader: ConfigLoader, node: yaml.nodes.ScalarNode - ) -> t.Any: - path = loader.base_path / loader.construct_scalar(node) - if not path.is_file(): - raise yaml.scanner.ScannerError( - "while processing 'include' tag", - None, - f"file {path} not found", - loader.get_mark(), # type: ignore - ) - with path.open() as fd: - return yaml.load(fd, config_loader(path.parent)) - - ConfigLoader.add_constructor("!env", tag_env) - ConfigLoader.add_constructor("!file", tag_file) - ConfigLoader.add_constructor("!include", tag_include) + with path.open() as fd: + return yaml.load(fd, _config_loader(path.parent)) + + +class _ConfigLoader(yaml.SafeLoader): + """YAML loader supporting tags.""" + + base_path: t.ClassVar[Path] + + +def _config_loader(path: Path) -> type[_ConfigLoader]: + class ConfigLoaderWithPath(_ConfigLoader): + base_path = path + + return ConfigLoaderWithPath + +def _tag_env(loader: _ConfigLoader, node: yaml.nodes.ScalarNode) -> t.Any: + env = loader.construct_scalar(node) + value = os.getenv(env) + if value is None: + raise yaml.scanner.ScannerError( + "while processing 'env' tag", + None, + f"variable {env} undefined", + loader.get_mark(), # type: ignore + ) + return yaml.safe_load(value) + + +def _tag_file(loader: _ConfigLoader, node: yaml.nodes.ScalarNode) -> str: + path = loader.base_path / loader.construct_scalar(node) + if not path.is_file(): + raise yaml.scanner.ScannerError( + "while processing 'file' tag", + None, + f"file {path} not found", + loader.get_mark(), # type: ignore + ) + return path.read_text().strip() + + +def _tag_include(loader: _ConfigLoader, node: yaml.nodes.ScalarNode) -> t.Any: + path = loader.base_path / loader.construct_scalar(node) + if not path.is_file(): + raise yaml.scanner.ScannerError( + "while processing 'include' tag", + None, + f"file {path} not found", + loader.get_mark(), # type: ignore + ) with path.open() as fd: - return yaml.load(fd, config_loader(path.parent)) + return yaml.load(fd, _config_loader(path.parent)) + + +_ConfigLoader.add_constructor("!env", _tag_env) +_ConfigLoader.add_constructor("!file", _tag_file) +_ConfigLoader.add_constructor("!include", _tag_include) From 4981404779f99ada14d292b752da6c49651a26ec Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 2 Jan 2025 12:51:05 +0100 Subject: [PATCH 069/110] chore: add table documenting exporter options --- README.rst | 107 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 44 deletions(-) diff --git a/README.rst b/README.rst index 6cf3cb7..f38d2c6 100644 --- a/README.rst +++ b/README.rst @@ -20,8 +20,8 @@ The application is simply run as:: which will look for a ``config.yaml`` configuration file in the current directory, containing the definitions of the databases to connect and queries to perform to update metrics. The configuration file can be overridden by -passing the ``--config`` option, or setting the ``QE_CONFIG`` environment -variable. +passing the ``--config`` option (or setting the ``QE_CONFIG`` environment +variable). A sample configuration file for the application looks like this: @@ -94,6 +94,36 @@ A sample configuration file for the application looks like this: See the `configuration file format`_ documentation for complete details on availble configuration options. + +Exporter options +---------------- + +The exporter provides the following options, that can be set via command-line +switches, environment variables or through the ``.env`` file: + +.. table:: + :widths: auto + + ====================== ====================== =================== ============================================================== + Command-line option Environment variable Default Description + ====================== ====================== =================== ============================================================== + ``-H``, ``--host`` ``QE_HOST`` ``localhost`` host addresses to bind. Multiple values can be provided. + ``-p``, ``--port`` ``QE_PORT`` ``9560`` Port to run the webserver on. + ``--metrics-path`` ``QE_METRICS_PATH`` ``/metrics`` Path under which metrics are exposed. + ``-L``, ``--log-level`` ``QE_LOG_LEVEL`` ``info`` Minimum level for log messages level. + One of ``critical``, ``error``, ``warning``, ``info``, ``debug``. + ``--log-format`` ``QE_LOG_FORMAT`` ``plain`` Log output format. One of ``plain``, ``json``. + ``--process-stats`` ``QE_PROCESS_STATS`` ``false`` Include process stats in metrics. + ``--ssl-private-key`` ``QE_SSL_PRIVATE_KEY`` Full path to the SSL private key. + ``--ssl-public-key`` ``QE_SSL_PUBLIC_KEY`` Full path to the SSL public key. + ``--ssl-ca`` ``QE_SSL_CA`` Full path to the SSL certificate authority (CA). + ``--check-only`` ``QE_CHECK_ONLY`` ``false`` Only check configuration, don't run the exporter. + ``--config`` ``QE_CONFIG`` ``config.yaml`` Configuration file. + ``QE_DOTENV`` ``$PWD/.env`` Path for the dotenv file where environment variables can be + provided. + ====================== ====================== =================== ============================================================== + + Metrics endpoint ---------------- @@ -104,9 +134,6 @@ endpoint. By default, the port is bound on ``localhost``. Note that if the name resolves both IPv4 and IPv6 addressses, the exporter will bind on both. -Both port and host can be overridden via command-line parameters and -environment variables. - Builtin metrics --------------- @@ -129,14 +156,6 @@ In addition, metrics for resources usage for the exporter process can be included by passing ``--process-stats`` in the command line. -Debugging / Logs ----------------- - -You can enable extended logging using the ``-L`` (or ``--log-level``) command -line switch. Possible log levels are ``critical``, ``error``, ``warning``, -``info``, ``debug``. - - Database engines ---------------- @@ -151,37 +170,6 @@ based on which database engines are needed. See `supported databases`_ for details. -Install from Snap ------------------ - -|Get it from the Snap Store| - -``query-exporter`` can be installed from `Snap Store`_ on systems where Snaps -are supported, via:: - - sudo snap install query-exporter - -The snap provides both the ``query-exporter`` command and a daemon instance of -the command, managed via a Systemd service. - -To configure the daemon: - -- create or edit ``/var/snap/query-exporter/current/config.yaml`` with the - configuration -- optionally, create a ``/var/snap/query-exporter/current/.env`` file with - environment variables definitions for additional config options -- run ``sudo snap restart query-exporter`` - -The snap has support for connecting the following databases: - -- PostgreSQL (``postgresql://``) -- MySQL (``mysql://``) -- SQLite (``sqlite://``) -- Microsoft SQL Server (``mssql://``) -- IBM DB2 (``db2://``) on supported architectures (x86_64, ppc64le and - s390x) - - Run in Docker ------------- @@ -225,6 +213,37 @@ by passing ``--build-arg ODBC_bVERSION_NUMBER``, e.g.:: docker build . --build-arg ODBC_DRIVER_VERSION=17 +Install from Snap +----------------- + +|Get it from the Snap Store| + +``query-exporter`` can be installed from `Snap Store`_ on systems where Snaps +are supported, via:: + + sudo snap install query-exporter + +The snap provides both the ``query-exporter`` command and a daemon instance of +the command, managed via a Systemd service. + +To configure the daemon: + +- create or edit ``/var/snap/query-exporter/current/config.yaml`` with the + configuration +- optionally, create a ``/var/snap/query-exporter/current/.env`` file with + environment variables definitions for additional config options +- run ``sudo snap restart query-exporter`` + +The snap has support for connecting the following databases: + +- PostgreSQL (``postgresql://``) +- MySQL (``mysql://``) +- SQLite (``sqlite://``) +- Microsoft SQL Server (``mssql://``) +- IBM DB2 (``db2://``) on supported architectures (x86_64, ppc64le and + s390x) + + .. _Prometheus: https://prometheus.io/ .. _SQLAlchemy: https://www.sqlalchemy.org/ .. _`supported databases`: From 7c3176e096780bdfb2f918770ad1a4e885dca9ec Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 2 Jan 2025 23:33:14 +0100 Subject: [PATCH 070/110] chore: better typing --- query_exporter/db.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/query_exporter/db.py b/query_exporter/db.py index 5b4b90f..604af7f 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -33,7 +33,6 @@ Connection, CursorResult, Engine, - Row, ) from sqlalchemy.exc import ( ArgumentError, @@ -159,7 +158,7 @@ class QueryResults(t.NamedTuple): """Results of a database query.""" keys: list[str] - rows: Sequence[Row[t.Any]] + rows: Sequence[Sequence[t.Any]] timestamp: float | None = None latency: float | None = None @@ -168,7 +167,7 @@ def from_result(cls, result: CursorResult[t.Any]) -> t.Self: """Return a QueryResults from results for a query.""" timestamp = time() keys: list[str] = [] - rows: Sequence[Row[t.Any]] = [] + rows: Sequence[Sequence[t.Any]] = [] if result.returns_rows: keys, rows = list(result.keys()), result.all() latency = result.connection.info.get("query_latency", None) From c404f2c0bf26ce491127d0340a29fe48724d984a Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 3 Jan 2025 10:51:26 +0100 Subject: [PATCH 071/110] feat: support combining config from multiple files (fixes #108) (#218) --- query_exporter/config.py | 46 +++++++---- query_exporter/db.py | 35 ++++---- query_exporter/main.py | 7 +- tests/config_test.py | 170 +++++++++++++++++++++++++++++---------- tests/loop_test.py | 2 +- 5 files changed, 179 insertions(+), 81 deletions(-) diff --git a/query_exporter/config.py b/query_exporter/config.py index 7687fba..438fd8f 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -2,7 +2,7 @@ from collections import defaultdict from collections.abc import Mapping -from dataclasses import dataclass +import dataclasses from functools import reduce from importlib import resources import itertools @@ -81,7 +81,7 @@ class ConfigError(Exception): """Configuration is invalid.""" -@dataclass(frozen=True) +@dataclasses.dataclass(frozen=True) class Config: """Top-level configuration.""" @@ -98,7 +98,7 @@ class Config: def load_config( - config_path: Path, + paths: list[Path], logger: structlog.stdlib.BoundLogger | None = None, env: Environ = os.environ, ) -> Config: @@ -107,7 +107,7 @@ def load_config( logger = structlog.get_logger() try: - data = defaultdict(dict, load_yaml_config(config_path)) + data = _load_config(paths) except yaml.scanner.ScannerError as e: raise ConfigError(str(e)) _validate_config(data) @@ -122,6 +122,34 @@ def load_config( return config +def _load_config(paths: list[Path]) -> dict[str, t.Any]: + """Return the combined configuration from provided files.""" + config: dict[str, t.Any] = defaultdict(dict) + for path in paths: + data = defaultdict(dict, load_yaml_config(path)) + for key in (field.name for field in dataclasses.fields(Config)): + entries = data.pop(key, None) + if entries is not None: + if overlap_entries := set(config[key]) & set(entries): + overlap_list = ", ".join(sorted(overlap_entries)) + raise ConfigError( + f'Duplicated entries in the "{key}" section: {overlap_list}' + ) + config[key].update(entries) + config.update(data) + return config + + +def _validate_config(config: dict[str, t.Any]) -> None: + schema_file = resources.files("query_exporter") / "schemas" / "config.yaml" + schema = yaml.safe_load(schema_file.read_bytes()) + try: + jsonschema.validate(config, schema) + except jsonschema.ValidationError as e: + path = "/".join(str(item) for item in e.absolute_path) + raise ConfigError(f"Invalid config at {path}: {e.message}") + + def _get_databases( configs: dict[str, dict[str, t.Any]], env: Environ ) -> tuple[dict[str, DataBaseConfig], frozenset[str]]: @@ -389,16 +417,6 @@ def _build_dsn(details: dict[str, t.Any]) -> str: return url -def _validate_config(config: dict[str, t.Any]) -> None: - schema_file = resources.files("query_exporter") / "schemas" / "config.yaml" - schema = yaml.safe_load(schema_file.read_bytes()) - try: - jsonschema.validate(config, schema) - except jsonschema.ValidationError as e: - path = "/".join(str(item) for item in e.absolute_path) - raise ConfigError(f"Invalid config at {path}: {e.message}") - - def _warn_if_unused( config: Config, logger: structlog.stdlib.BoundLogger ) -> None: diff --git a/query_exporter/db.py b/query_exporter/db.py index 604af7f..e82c1f5 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -190,30 +190,23 @@ class MetricResults(t.NamedTuple): latency: float | None = None +@dataclass class Query: """Query definition and configuration.""" - def __init__( - self, - name: str, - databases: list[str], - metrics: list[QueryMetric], - sql: str, - parameters: dict[str, t.Any] | None = None, - timeout: QueryTimeout | None = None, - interval: int | None = None, - schedule: str | None = None, - config_name: str | None = None, - ) -> None: - self.name = name - self.databases = databases - self.metrics = metrics - self.sql = sql - self.parameters = parameters or {} - self.timeout = timeout - self.interval = interval - self.schedule = schedule - self.config_name = config_name or name + name: str + databases: list[str] + metrics: list[QueryMetric] + sql: str + parameters: dict[str, t.Any] = field(default_factory=dict) + timeout: QueryTimeout | None = None + interval: int | None = None + schedule: str | None = None + config_name: str = "" + + def __post_init__(self) -> None: + if not self.config_name: + self.config_name = self.name self._check_schedule() self._check_query_parameters() diff --git a/query_exporter/main.py b/query_exporter/main.py index c129f69..0873df4 100644 --- a/query_exporter/main.py +++ b/query_exporter/main.py @@ -48,7 +48,8 @@ def command_line_parameters(self) -> list[click.Parameter]: path_type=Path, ), help="configuration file", - default=Path("config.yaml"), + multiple=True, + default=[Path("config.yaml")], show_default=True, show_envvar=True, ), @@ -78,10 +79,10 @@ async def _update_handler( await query_loop.run_aperiodic_queries() query_loop.clear_expired_series() - def _load_config(self, config_file: Path) -> Config: + def _load_config(self, paths: list[Path]) -> Config: """Load the application configuration.""" try: - return load_config(config_file, self.logger) + return load_config(paths, self.logger) except (InvalidMetricType, ConfigError) as error: self.logger.error("invalid config", error=str(error)) raise SystemExit(1) diff --git a/tests/config_test.py b/tests/config_test.py index e2d9085..20f0043 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -1,6 +1,7 @@ from collections.abc import Callable, Iterator from pathlib import Path import typing as t +import uuid import pytest from pytest_structlog import StructuredLogCapture @@ -28,7 +29,7 @@ def config_full() -> Iterator[dict[str, t.Any]]: "interval": 10, "databases": ["db"], "metrics": ["m"], - "sql": "SELECT 1 as m", + "sql": "SELECT 1 AS m", } }, } @@ -39,9 +40,8 @@ def config_full() -> Iterator[dict[str, t.Any]]: @pytest.fixture def write_config(tmp_path: Path) -> Iterator[ConfigWriter]: - path = tmp_path / "config" - def write(data: dict[str, t.Any]) -> Path: + path = tmp_path / f"{uuid.uuid4()}.yaml" path.write_text(yaml.dump(data), "utf-8") return path @@ -129,10 +129,10 @@ def write(data: dict[str, t.Any]) -> Path: class TestLoadConfig: def test_load_invalid_format(self, tmp_path: Path) -> None: - config_file = tmp_path / "config" + config_file = tmp_path / "invalid.yaml" config_file.write_text("foo: !env UNSET") with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert "variable UNSET undefined" in str(err.value) def test_load_databases_section(self, write_config: ConfigWriter) -> None: @@ -149,7 +149,7 @@ def test_load_databases_section(self, write_config: ConfigWriter) -> None: "queries": {}, } config_file = write_config(cfg) - config = load_config(config_file) + config = load_config([config_file]) assert {"db1", "db2"} == set(config.databases) database1 = config.databases["db1"] database2 = config.databases["db2"] @@ -173,7 +173,7 @@ def test_load_databases_dsn_from_env( "queries": {}, } config_file = write_config(cfg) - config = load_config(config_file, env={"FOO": "sqlite://"}) + config = load_config([config_file], env={"FOO": "sqlite://"}) assert config.databases["db1"].dsn == "sqlite://" assert log.has( "deprecated DSN source 'env:FOO', use '!env FOO' instead" @@ -189,7 +189,7 @@ def test_load_databases_missing_dsn( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == "Invalid config at databases/db1: 'dsn' is a required property" @@ -205,7 +205,7 @@ def test_load_databases_invalid_dsn( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert str(err.value) == 'Invalid database DSN: "invalid"' def test_load_databases_dsn_details( @@ -224,7 +224,7 @@ def test_load_databases_dsn_details( "queries": {}, } config_file = write_config(cfg) - config = load_config(config_file) + config = load_config([config_file]) assert config.databases["db1"].dsn == "sqlite:///path/to/file" def test_load_databases_dsn_details_only_dialect( @@ -242,7 +242,7 @@ def test_load_databases_dsn_details_only_dialect( "queries": {}, } config_file = write_config(cfg) - config = load_config(config_file) + config = load_config([config_file]) assert config.databases["db1"].dsn == "sqlite://" def test_load_databases_dsn_invalid_env( @@ -255,7 +255,7 @@ def test_load_databases_dsn_invalid_env( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert str(err.value) == 'Invalid variable name: "NOT-VALID"' def test_load_databases_dsn_undefined_env( @@ -268,7 +268,7 @@ def test_load_databases_dsn_undefined_env( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file, env={}) + load_config([config_file], env={}) assert str(err.value) == 'Undefined variable: "FOO"' def test_load_databases_dsn_from_file( @@ -286,7 +286,7 @@ def test_load_databases_dsn_from_file( "queries": {}, } config_file = write_config(cfg) - config = load_config(config_file) + config = load_config([config_file]) assert config.databases["db1"].dsn == dsn assert log.has( f"deprecated DSN source 'file:{dsn_path}', use '!file {dsn_path}' instead" @@ -302,7 +302,7 @@ def test_load_databases_dsn_from_file_not_found( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == 'Unable to read dsn file : "/not/found": No such file or directory' @@ -321,7 +321,7 @@ def test_load_databases_no_labels( "queries": {}, } config_file = write_config(cfg) - result = load_config(config_file) + result = load_config([config_file]) db = result.databases["db"] assert db.labels == {} @@ -337,7 +337,7 @@ def test_load_databases_labels(self, write_config: ConfigWriter) -> None: "queries": {}, } config_file = write_config(cfg) - result = load_config(config_file) + result = load_config([config_file]) db = result.databases["db"] assert db.labels == {"label1": "value1", "label2": "value2"} @@ -360,7 +360,7 @@ def test_load_databases_labels_not_all_same( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file, env={}) + load_config([config_file], env={}) assert str(err.value) == "Not all databases define the same labels" def test_load_databases_connect_sql( @@ -377,7 +377,7 @@ def test_load_databases_connect_sql( "queries": {}, } config_file = write_config(cfg) - result = load_config(config_file, env={}) + result = load_config([config_file], env={}) assert result.databases["db"].connect_sql == ["SELECT 1", "SELECT 2"] def test_load_metrics_section(self, write_config: ConfigWriter) -> None: @@ -405,7 +405,7 @@ def test_load_metrics_section(self, write_config: ConfigWriter) -> None: "queries": {}, } config_file = write_config(cfg) - result = load_config(config_file) + result = load_config([config_file]) metric1 = result.metrics["metric1"] assert metric1.type == "summary" assert metric1.description == "metric one" @@ -441,7 +441,7 @@ def test_load_metrics_overlap_reserved_label( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == 'Labels for metric "m" overlap with reserved/database ones: database' @@ -457,7 +457,7 @@ def test_load_metrics_overlap_database_label( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == 'Labels for metric "m" overlap with reserved/database ones: l1' @@ -473,7 +473,7 @@ def test_load_metrics_reserved_name( config_full["metrics"][global_name] = {"type": "counter"} config_file = write_config(config_full) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == f'Label name "{global_name} is reserved for builtin metric' @@ -491,7 +491,7 @@ def test_load_metrics_unsupported_type( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert str(err.value) == ( "Invalid config at metrics/metric1/type: 'info' is not one of " "['counter', 'enum', 'gauge', 'histogram', 'summary']" @@ -523,7 +523,7 @@ def test_load_queries_section(self, write_config: ConfigWriter) -> None: }, } config_file = write_config(cfg) - result = load_config(config_file) + result = load_config([config_file]) query1 = result.queries["q1"] assert query1.name == "q1" assert query1.databases == ["db1"] @@ -557,7 +557,7 @@ def test_load_queries_section_with_parameters( }, } config_file = write_config(cfg) - result = load_config(config_file) + result = load_config([config_file]) query1 = result.queries["q[params0]"] assert query1.name == "q[params0]" assert query1.databases == ["db"] @@ -597,7 +597,7 @@ def test_load_queries_section_with_parameters_matrix( }, } config_file = write_config(cfg) - result = load_config(config_file) + result = load_config([config_file]) assert len(result.queries) == 4 @@ -660,7 +660,7 @@ def test_load_queries_section_with_wrong_parameters( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == 'Parameters for query "q[params0]" don\'t match those from SQL' @@ -684,7 +684,7 @@ def test_load_queries_section_with_schedule_and_interval( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == 'Invalid schedule for query "q": both interval and schedule specified' @@ -707,7 +707,7 @@ def test_load_queries_section_invalid_schedule( } config_file = write_config(cfg) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == 'Invalid schedule for query "q": invalid schedule format' @@ -720,7 +720,7 @@ def test_load_queries_section_timeout( ) -> None: config_full["queries"]["q"]["timeout"] = 2.0 config_file = write_config(config_full) - result = load_config(config_file) + result = load_config([config_file]) query1 = result.queries["q"] assert query1.timeout == 2.0 @@ -751,7 +751,7 @@ def test_load_queries_section_invalid_timeout( config_full["queries"]["q"]["timeout"] = timeout config_file = write_config(config_full) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert str(err.value) == error_message @pytest.mark.parametrize( @@ -797,7 +797,7 @@ def test_configuration_incorrect( ) -> None: config_file = write_config(config) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert str(err.value) == error_message def test_configuration_warning_unused( @@ -811,7 +811,7 @@ def test_configuration_warning_unused( config_full["metrics"]["m2"] = {"type": "gauge"} config_full["metrics"]["m3"] = {"type": "gauge"} config_file = write_config(config_full) - load_config(config_file) + load_config([config_file]) assert log.has( "unused config entries", section="databases", @@ -832,7 +832,7 @@ def test_load_queries_missing_interval_default_to_none( }, } config_file = write_config(cfg) - config = load_config(config_file) + config = load_config([config_file]) assert config.queries["q"].interval is None @pytest.mark.parametrize( @@ -856,7 +856,7 @@ def test_load_queries_interval( ) -> None: config_full["queries"]["q"]["interval"] = interval config_file = write_config(config_full) - config = load_config(config_file) + config = load_config([config_file]) assert config.queries["q"].interval == value def test_load_queries_interval_not_specified( @@ -866,7 +866,7 @@ def test_load_queries_interval_not_specified( ) -> None: del config_full["queries"]["q"]["interval"] config_file = write_config(config_full) - config = load_config(config_file) + config = load_config([config_file]) assert config.queries["q"].interval is None @pytest.mark.parametrize("interval", ["1x", "wrong", "1.5m"]) @@ -879,7 +879,7 @@ def test_load_queries_invalid_interval_string( config_full["queries"]["q"]["interval"] = interval config_file = write_config(config_full) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert str(err.value) == ( "Invalid config at queries/q/interval: " f"'{interval}' does not match '^[0-9]+[smhd]?$'" @@ -895,7 +895,7 @@ def test_load_queries_invalid_interval_number( config_full["queries"]["q"]["interval"] = interval config_file = write_config(config_full) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == f"Invalid config at queries/q/interval: {interval} is less than the minimum of 1" @@ -909,7 +909,7 @@ def test_load_queries_no_metrics( config_full["queries"]["q"]["metrics"] = [] config_file = write_config(config_full) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == "Invalid config at queries/q/metrics: [] should be non-empty" @@ -923,7 +923,7 @@ def test_load_queries_no_databases( config_full["queries"]["q"]["databases"] = [] config_file = write_config(config_full) with pytest.raises(ConfigError) as err: - load_config(config_file) + load_config([config_file]) assert ( str(err.value) == "Invalid config at queries/q/databases: [] should be non-empty" @@ -950,9 +950,95 @@ def test_load_metrics_expiration( ) -> None: config_full["metrics"]["m"]["expiration"] = expiration config_file = write_config(config_full) - config = load_config(config_file) + config = load_config([config_file]) assert config.metrics["m"].config["expiration"] == value + def test_load_multiple_files( + self, + config_full: dict[str, t.Any], + write_config: ConfigWriter, + ) -> None: + file_full = write_config(config_full) + file1 = write_config({"databases": config_full["databases"]}) + file2 = write_config({"metrics": config_full["metrics"]}) + file3 = write_config({"queries": config_full["queries"]}) + assert load_config([file1, file2, file3]) == load_config([file_full]) + + def test_load_multiple_files_combine( + self, + write_config: ConfigWriter, + ) -> None: + file1 = write_config( + { + "databases": {"db1": {"dsn": "sqlite://"}}, + "metrics": {"m1": {"type": "gauge"}}, + "queries": { + "q1": { + "databases": ["db1"], + "metrics": ["m1"], + "sql": "SELECT 1 AS m1", + } + }, + } + ) + file2 = write_config( + { + "databases": {"db2": {"dsn": "sqlite://"}}, + "metrics": {"m2": {"type": "gauge"}}, + "queries": { + "q2": { + "databases": ["db2"], + "metrics": ["m2"], + "sql": "SELECT 2 AS m2", + } + }, + } + ) + config = load_config([file1, file2]) + assert set(config.databases) == {"db1", "db2"} + assert set(config.metrics) == {"m1", "m2"} | GLOBAL_METRICS + assert set(config.queries) == {"q1", "q2"} + + def test_load_multiple_files_duplicated_database( + self, + config_full: dict[str, t.Any], + write_config: ConfigWriter, + ) -> None: + file1 = write_config(config_full) + file2 = write_config({"databases": config_full["databases"]}) + with pytest.raises(ConfigError) as err: + load_config([file1, file2]) + assert ( + str(err.value) + == 'Duplicated entries in the "databases" section: db' + ) + + def test_load_multiple_files_duplicated_metric( + self, + config_full: dict[str, t.Any], + write_config: ConfigWriter, + ) -> None: + file1 = write_config(config_full) + file2 = write_config({"metrics": config_full["metrics"]}) + with pytest.raises(ConfigError) as err: + load_config([file1, file2]) + assert ( + str(err.value) == 'Duplicated entries in the "metrics" section: m' + ) + + def test_load_multiple_files_duplicated_query( + self, + config_full: dict[str, t.Any], + write_config: ConfigWriter, + ) -> None: + file1 = write_config(config_full) + file2 = write_config({"queries": config_full["queries"]}) + with pytest.raises(ConfigError) as err: + load_config([file1, file2]) + assert ( + str(err.value) == 'Duplicated entries in the "queries" section: q' + ) + class TestResolveDSN: def test_all_details(self) -> None: diff --git a/tests/loop_test.py b/tests/loop_test.py index 31ca22f..621df29 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -54,7 +54,7 @@ async def make_query_loop( def make_loop() -> loop.QueryLoop: config_file = tmp_path / "config.yaml" config_file.write_text(yaml.dump(config_data), "utf-8") - config = load_config(config_file) + config = load_config([config_file]) registry.create_metrics(config.metrics.values()) query_loop = loop.QueryLoop(config, registry) query_loops.append(query_loop) From 700e35835267008fb83cfc6588b83bb6d1d04e1d Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 3 Jan 2025 11:02:24 +0100 Subject: [PATCH 072/110] chore: error if file content is not a mapping --- query_exporter/config.py | 5 ++++- tests/config_test.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/query_exporter/config.py b/query_exporter/config.py index 438fd8f..a76b377 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -126,7 +126,10 @@ def _load_config(paths: list[Path]) -> dict[str, t.Any]: """Return the combined configuration from provided files.""" config: dict[str, t.Any] = defaultdict(dict) for path in paths: - data = defaultdict(dict, load_yaml_config(path)) + conf = load_yaml_config(path) + if not isinstance(conf, dict): + raise ConfigError(f"File content is not a mapping: {path}") + data = defaultdict(dict, conf) for key in (field.name for field in dataclasses.fields(Config)): entries = data.pop(key, None) if entries is not None: diff --git a/tests/config_test.py b/tests/config_test.py index 20f0043..3ccf1cd 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -128,13 +128,23 @@ def write(data: dict[str, t.Any]) -> Path: class TestLoadConfig: - def test_load_invalid_format(self, tmp_path: Path) -> None: + def test_load_invalid(self, tmp_path: Path) -> None: config_file = tmp_path / "invalid.yaml" config_file.write_text("foo: !env UNSET") with pytest.raises(ConfigError) as err: load_config([config_file]) assert "variable UNSET undefined" in str(err.value) + def test_load_not_mapping( + self, tmp_path: Path, write_config: ConfigWriter + ) -> None: + config_file = write_config(["a", "b", "c"]) + with pytest.raises(ConfigError) as err: + load_config([config_file]) + assert ( + str(err.value) == f"File content is not a mapping: {config_file}" + ) + def test_load_databases_section(self, write_config: ConfigWriter) -> None: cfg = { "databases": { From 1aa22e8a56c34fef6c4c2adc453476e10e9481a4 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 4 Jan 2025 09:27:24 +0100 Subject: [PATCH 073/110] chore: update doc --- README.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index f38d2c6..6a740e4 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,9 @@ which will look for a ``config.yaml`` configuration file in the current directory, containing the definitions of the databases to connect and queries to perform to update metrics. The configuration file can be overridden by passing the ``--config`` option (or setting the ``QE_CONFIG`` environment -variable). +variable). The option can be provided multiple times to pass partial +configuration files, the resulting configuration will be the merge of the +content of each top-level section (``databases``, ``metrics``, ``queries``). A sample configuration file for the application looks like this: @@ -118,7 +120,7 @@ switches, environment variables or through the ``.env`` file: ``--ssl-public-key`` ``QE_SSL_PUBLIC_KEY`` Full path to the SSL public key. ``--ssl-ca`` ``QE_SSL_CA`` Full path to the SSL certificate authority (CA). ``--check-only`` ``QE_CHECK_ONLY`` ``false`` Only check configuration, don't run the exporter. - ``--config`` ``QE_CONFIG`` ``config.yaml`` Configuration file. + ``--config`` ``QE_CONFIG`` ``config.yaml`` Configuration files. Multiple values can be provided. ``QE_DOTENV`` ``$PWD/.env`` Path for the dotenv file where environment variables can be provided. ====================== ====================== =================== ============================================================== From 777e689f7d3697e7f6600ded6e74a2163983de37 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 4 Jan 2025 10:01:26 +0100 Subject: [PATCH 074/110] chore: fix table in doc --- README.rst | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index 6a740e4..a08a812 100644 --- a/README.rst +++ b/README.rst @@ -106,24 +106,35 @@ switches, environment variables or through the ``.env`` file: .. table:: :widths: auto - ====================== ====================== =================== ============================================================== - Command-line option Environment variable Default Description - ====================== ====================== =================== ============================================================== - ``-H``, ``--host`` ``QE_HOST`` ``localhost`` host addresses to bind. Multiple values can be provided. - ``-p``, ``--port`` ``QE_PORT`` ``9560`` Port to run the webserver on. - ``--metrics-path`` ``QE_METRICS_PATH`` ``/metrics`` Path under which metrics are exposed. - ``-L``, ``--log-level`` ``QE_LOG_LEVEL`` ``info`` Minimum level for log messages level. - One of ``critical``, ``error``, ``warning``, ``info``, ``debug``. - ``--log-format`` ``QE_LOG_FORMAT`` ``plain`` Log output format. One of ``plain``, ``json``. - ``--process-stats`` ``QE_PROCESS_STATS`` ``false`` Include process stats in metrics. - ``--ssl-private-key`` ``QE_SSL_PRIVATE_KEY`` Full path to the SSL private key. - ``--ssl-public-key`` ``QE_SSL_PUBLIC_KEY`` Full path to the SSL public key. - ``--ssl-ca`` ``QE_SSL_CA`` Full path to the SSL certificate authority (CA). - ``--check-only`` ``QE_CHECK_ONLY`` ``false`` Only check configuration, don't run the exporter. - ``--config`` ``QE_CONFIG`` ``config.yaml`` Configuration files. Multiple values can be provided. - ``QE_DOTENV`` ``$PWD/.env`` Path for the dotenv file where environment variables can be - provided. - ====================== ====================== =================== ============================================================== + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | Command-line option | Environment variable | Default | Description | + +=========================+========================+=================+===================================================================+ + | ``-H``, ``--host`` | ``QE_HOST`` | ``localhost`` | Host addresses to bind. Multiple values can be provided. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | ``-p``, ``--port`` | ``QE_PORT`` | ``9560`` | Port to run the webserver on. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | ``--metrics-path`` | ``QE_METRICS_PATH`` | ``/metrics`` | Path under which metrics are exposed. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | ``-L``, ``--log-level`` | ``QE_LOG_LEVEL`` | ``info`` | Minimum level for log messages level. | + | | | | One of ``critical``, ``error``, ``wanring``, ``info``, ``debug``. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | ``--log-format`` | ``QE_LOG_FORMAT`` | ``plain`` | Log output format. One of ``plain``, ``json``. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | ``--process-stats`` | ``QE_PROCESS_STATS`` | ``false`` | Include process stats in metrics. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | ``--ssl-private-key`` | ``QE_SSL_PRIVATE_KEY`` | | Full path to the SSL private key. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | ``--ssl-public-key`` | ``QE_SSL_PUBLIC_KEY`` | | Full path to the SSL public key. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | ``--ssl-ca`` | ``QE_SSL_CA`` | | Full path to the SSL certificate authority (CA). | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | ``--check-only`` | ``QE_CHECK_ONLY`` | ``false`` | Only check configuration, don't run the exporter. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | ``--config`` | ``QE_CONFIG`` | ``config.yaml`` | Configuration files. Multiple values can be provided. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ + | | ``QE_DOTENV`` | ``$PWD/.env`` | Path for the dotenv file where environment variables can be | + | | | | provided. | + +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ From ffc00b238a2007d5c88b5cd1903c9c1275d1a30d Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 4 Jan 2025 17:54:07 +0100 Subject: [PATCH 075/110] chore: more tests typing fixes --- tests/config_test.py | 4 ++-- tests/db_test.py | 31 ++++++++++++++++--------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/config_test.py b/tests/config_test.py index 3ccf1cd..cd57a5a 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -35,12 +35,12 @@ def config_full() -> Iterator[dict[str, t.Any]]: } -ConfigWriter = Callable[[dict[str, t.Any]], Path] +ConfigWriter = Callable[[t.Any], Path] @pytest.fixture def write_config(tmp_path: Path) -> Iterator[ConfigWriter]: - def write(data: dict[str, t.Any]) -> Path: + def write(data: t.Any) -> Path: path = tmp_path / f"{uuid.uuid4()}.yaml" path.write_text(yaml.dump(data), "utf-8") return path diff --git a/tests/db_test.py b/tests/db_test.py index 0b82dc8..45d3278 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -1,5 +1,6 @@ import asyncio -from collections.abc import Iterator +from collections.abc import AsyncIterator, Iterator +from threading import Thread import time import typing as t from unittest.mock import ANY @@ -298,7 +299,7 @@ def test_from_result(self) -> None: assert query_results.keys == ["a", "b"] assert query_results.rows == [(1, 2)] assert query_results.latency is None - assert query_results.timestamp < time.time() + assert t.cast(float, query_results.timestamp) < time.time() def test_from_empty(self) -> None: engine = create_engine("sqlite://") @@ -319,11 +320,11 @@ def test_from_result_with_latency(self) -> None: assert query_results.keys == ["a", "b"] assert query_results.rows == [(1, 2)] assert query_results.latency == 1.2 - assert query_results.timestamp < time.time() + assert t.cast(float, query_results.timestamp) < time.time() @pytest.fixture -async def conn() -> Iterator[DataBaseConnection]: +async def conn() -> AsyncIterator[DataBaseConnection]: engine = create_engine("sqlite://") connection = DataBaseConnection("db", engine) yield connection @@ -358,7 +359,7 @@ async def test_open(self, conn: DataBaseConnection) -> None: await conn.open() assert conn.connected assert conn._conn is not None - assert conn._worker.is_alive() + assert t.cast(Thread, conn._worker).is_alive() async def test_open_noop(self, conn: DataBaseConnection) -> None: await conn.open() @@ -401,7 +402,7 @@ def db_config() -> Iterator[DataBaseConfig]: @pytest.fixture -async def db(db_config: DataBaseConfig) -> Iterator[DataBase]: +async def db(db_config: DataBaseConfig) -> AsyncIterator[DataBase]: db = DataBase(db_config) yield db await db.close() @@ -446,7 +447,7 @@ async def test_connect_error(self) -> None: await db.connect() assert "unable to open database file" in str(error.value) - async def test_connect_sql(self) -> None: + async def test_connect_sql(self, mocker: MockerFixture) -> None: config = DataBaseConfig( name="db", dsn="sqlite://", @@ -456,10 +457,10 @@ async def test_connect_sql(self) -> None: queries = [] - async def execute_sql(sql): + async def execute_sql(sql: str) -> None: queries.append(sql) - db.execute_sql = execute_sql + mocker.patch.object(db, "execute_sql", execute_sql) await db.connect() assert queries == ["SELECT 1", "SELECT 2"] await db.close() @@ -608,7 +609,7 @@ async def test_execute_with_labels(self, db: DataBase) -> None: ] async def test_execute_fail(self, db: DataBase) -> None: - query = Query("query", 10, [QueryMetric("metric", [])], "WRONG") + query = Query("query", ["db"], [QueryMetric("metric", [])], "WRONG") await db.connect() with pytest.raises(DataBaseQueryError) as error: await db.execute(query) @@ -619,7 +620,7 @@ async def test_execute_query_invalid_count( ) -> None: query = Query( "query", - 20, + ["db"], [QueryMetric("metric", [])], "SELECT 1 AS metric, 2 AS other", ) @@ -686,7 +687,7 @@ async def test_execute_debug_exception( ) await db.connect() exception = Exception("boom!") - mocker.patch.object(db, "execute_sql").side_effect = exception + mocker.patch.object(db, "execute_sql", side_effect=exception) with pytest.raises(DataBaseQueryError) as error: await db.execute(query) @@ -701,7 +702,7 @@ async def test_execute_debug_exception( ) async def test_execute_timeout( - self, log: StructuredLogCapture, db: DataBase + self, mocker: MockerFixture, log: StructuredLogCapture, db: DataBase ) -> None: query = Query( "query", @@ -715,10 +716,10 @@ async def test_execute_timeout( async def execute( sql: TextClause, parameters: dict[str, t.Any] | None = None, - ) -> QueryResults: + ) -> None: await asyncio.sleep(1) # longer than timeout - db._conn.execute = execute + mocker.patch.object(db._conn, "execute", execute) with pytest.raises(QueryTimeoutExpired): await db.execute(query) From 11d8dfb008dd1bd8e8dd9b1769887445cb85f87c Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sun, 5 Jan 2025 09:47:44 +0100 Subject: [PATCH 076/110] chore: complete typing for tests --- tests/loop_test.py | 82 ++++++++++++++++++++++++++++------------------ tox.ini | 2 +- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/tests/loop_test.py b/tests/loop_test.py index 621df29..394e782 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -7,14 +7,16 @@ from unittest.mock import ANY from prometheus_aioexporter import MetricsRegistry +from prometheus_client.metrics import MetricWrapperBase import pytest from pytest_mock import MockerFixture from pytest_structlog import StructuredLogCapture +from sqlalchemy.sql.elements import TextClause import yaml -from query_exporter import loop from query_exporter.config import load_config from query_exporter.db import DataBase, DataBaseConfig +from query_exporter.loop import MetricsLastSeen, QueryLoop from .conftest import QueryTracker @@ -42,7 +44,7 @@ def registry() -> Iterator[MetricsRegistry]: yield MetricsRegistry() -MakeQueryLoop = Callable[[], loop.QueryLoop] +MakeQueryLoop = Callable[[], QueryLoop] @pytest.fixture @@ -51,12 +53,12 @@ async def make_query_loop( ) -> AsyncIterator[MakeQueryLoop]: query_loops = [] - def make_loop() -> loop.QueryLoop: + def make_loop() -> QueryLoop: config_file = tmp_path / "config.yaml" config_file.write_text(yaml.dump(config_data), "utf-8") config = load_config([config_file]) registry.create_metrics(config.metrics.values()) - query_loop = loop.QueryLoop(config, registry) + query_loop = QueryLoop(config, registry) query_loops.append(query_loop) return query_loop @@ -70,30 +72,33 @@ def make_loop() -> loop.QueryLoop: @pytest.fixture async def query_loop( make_query_loop: MakeQueryLoop, -) -> AsyncIterator[loop.QueryLoop]: +) -> AsyncIterator[QueryLoop]: yield make_query_loop() -MetricValues = list[int | float] | dict[tuple[str], list[int | float]] +MetricValues = list[int | float] | dict[tuple[str, ...], int | float] -def metric_values(metric, by_labels: tuple[str] = ()) -> MetricValues: +def metric_values( + metric: MetricWrapperBase, by_labels: tuple[str, ...] = () +) -> MetricValues: """Return values for the metric.""" if metric._type == "gauge": suffix = "" elif metric._type == "counter": suffix = "_total" - values = defaultdict(list) + values_by_label: dict[tuple[str, ...], int | float] = {} + values_by_suffix: dict[str, list[int | float]] = defaultdict(list) for sample_suffix, labels, value, *_ in metric._samples(): if sample_suffix == suffix: if by_labels: label_values = tuple(labels[label] for label in by_labels) - values[label_values] = value + values_by_label[label_values] = value else: - values[sample_suffix].append(value) + values_by_suffix[sample_suffix].append(value) - return values if by_labels else values[suffix] + return values_by_label if by_labels else values_by_suffix[suffix] async def run_queries(db_file: Path, *queries: str) -> None: @@ -105,7 +110,7 @@ async def run_queries(db_file: Path, *queries: str) -> None: class TestMetricsLastSeen: def test_update(self) -> None: - last_seen = loop.MetricsLastSeen({"m1": 50, "m2": 100}) + last_seen = MetricsLastSeen({"m1": 50, "m2": 100}) last_seen.update("m1", {"l1": "v1", "l2": "v2"}, 100) last_seen.update("m1", {"l1": "v3", "l2": "v4"}, 200) last_seen.update("other", {"l3": "v100"}, 300) @@ -117,12 +122,12 @@ def test_update(self) -> None: } def test_update_label_values_sorted_by_name(self) -> None: - last_seen = loop.MetricsLastSeen({"m1": 50}) + last_seen = MetricsLastSeen({"m1": 50}) last_seen.update("m1", {"l2": "v2", "l1": "v1"}, 100) assert last_seen._last_seen == {"m1": {("v1", "v2"): 100}} def test_expire_series_not_expired(self) -> None: - last_seen = loop.MetricsLastSeen({"m1": 50}) + last_seen = MetricsLastSeen({"m1": 50}) last_seen.update("m1", {"l1": "v1", "l2": "v2"}, 10) last_seen.update("m1", {"l1": "v3", "l2": "v4"}, 20) assert last_seen.expire_series(30) == {} @@ -134,7 +139,7 @@ def test_expire_series_not_expired(self) -> None: } def test_expire_series(self) -> None: - last_seen = loop.MetricsLastSeen({"m1": 50, "m2": 100}) + last_seen = MetricsLastSeen({"m1": 50, "m2": 100}) last_seen.update("m1", {"l1": "v1", "l2": "v2"}, 10) last_seen.update("m1", {"l1": "v3", "l2": "v4"}, 100) last_seen.update("m2", {"l3": "v100"}, 100) @@ -145,7 +150,7 @@ def test_expire_series(self) -> None: } def test_expire_no_labels(self) -> None: - last_seen = loop.MetricsLastSeen({"m1": 50}) + last_seen = MetricsLastSeen({"m1": 50}) last_seen.update("m1", {}, 10) expired = last_seen.expire_series(120) assert expired == {"m1": [()]} @@ -154,21 +159,26 @@ def test_expire_no_labels(self) -> None: class TestQueryLoop: async def test_start( - self, query_tracker: QueryTracker, query_loop + self, + query_tracker: QueryTracker, + query_loop: QueryLoop, ) -> None: await query_loop.start() timed_call = query_loop._timed_calls["q"] assert timed_call.running await query_tracker.wait_results() - async def test_stop(self, query_loop) -> None: + async def test_stop(self, query_loop: QueryLoop) -> None: await query_loop.start() timed_call = query_loop._timed_calls["q"] await query_loop.stop() assert not timed_call.running async def test_run_query( - self, query_tracker: QueryTracker, query_loop: loop.QueryLoop, registry + self, + query_tracker: QueryTracker, + query_loop: QueryLoop, + registry: MetricsRegistry, ) -> None: await query_loop.start() await query_tracker.wait_results() @@ -192,15 +202,18 @@ async def test_run_scheduled_query( ) -> None: event_loop = asyncio.get_running_loop() - def croniter(*args: t.Any) -> float: + def croniter(*args: t.Any) -> Iterator[float]: while True: # sync croniter time with the loop one yield event_loop.time() + 60 - mock_croniter = mocker.patch.object(loop, "croniter") + mock_croniter = mocker.patch("query_exporter.loop.croniter") mock_croniter.side_effect = croniter # ensure that both clocks advance in sync - mocker.patch.object(loop.time, "time", lambda: event_loop.time()) # type: ignore + mocker.patch( + "query_exporter.loop.time.time", + lambda: event_loop.time(), + ) del config_data["queries"]["q"]["interval"] config_data["queries"]["q"]["schedule"] = "*/2 * * * *" @@ -303,7 +316,9 @@ async def test_run_query_metrics_with_database_labels( } async def test_update_metric_decimal_value( - self, registry: MetricsRegistry, make_query_loop + self, + registry: MetricsRegistry, + make_query_loop: MakeQueryLoop, ) -> None: db = DataBase(DataBaseConfig(name="db", dsn="sqlite://")) query_loop = make_query_loop() @@ -317,7 +332,7 @@ async def test_run_query_log( self, log: StructuredLogCapture, query_tracker: QueryTracker, - query_loop: loop.QueryLoop, + query_loop: QueryLoop, ) -> None: await query_loop.start() await query_tracker.wait_queries() @@ -371,7 +386,7 @@ async def test_run_query_increase_db_error_count( query_tracker: QueryTracker, config_data: dict[str, t.Any], make_query_loop: MakeQueryLoop, - registry, + registry: MetricsRegistry, ) -> None: config_data["databases"]["db"]["dsn"] = "sqlite:////invalid" query_loop = make_query_loop() @@ -382,11 +397,11 @@ async def test_run_query_increase_db_error_count( async def test_run_query_increase_database_error_count( self, - mocker, + mocker: MockerFixture, query_tracker: QueryTracker, config_data: dict[str, t.Any], make_query_loop: MakeQueryLoop, - registry, + registry: MetricsRegistry, ) -> None: query_loop = make_query_loop() db = query_loop._databases["db"] @@ -402,7 +417,7 @@ async def test_run_query_increase_query_error_count( query_tracker: QueryTracker, config_data: dict[str, t.Any], make_query_loop: MakeQueryLoop, - registry, + registry: MetricsRegistry, ) -> None: config_data["queries"]["q"]["sql"] = "SELECT 100.0 AS a, 200.0 AS b" query_loop = make_query_loop() @@ -415,10 +430,11 @@ async def test_run_query_increase_query_error_count( async def test_run_query_increase_timeout_count( self, + mocker: MockerFixture, query_tracker: QueryTracker, config_data: dict[str, t.Any], make_query_loop: MakeQueryLoop, - registry, + registry: MetricsRegistry, ) -> None: config_data["queries"]["q"]["timeout"] = 0.1 query_loop = make_query_loop() @@ -426,10 +442,12 @@ async def test_run_query_increase_timeout_count( db = query_loop._databases["db"] await db.connect() - async def execute(sql, parameters): + async def execute( + sql: TextClause, parameters: dict[str, t.Any] | None + ) -> None: await asyncio.sleep(1) # longer than timeout - db._conn.execute = execute + mocker.patch.object(db._conn, "execute", execute) await query_tracker.wait_failures() queries_metric = registry.get_metric("queries") @@ -441,7 +459,7 @@ async def test_run_query_at_interval( self, advance_time: AdvanceTime, query_tracker: QueryTracker, - query_loop: loop.QueryLoop, + query_loop: QueryLoop, ) -> None: await query_loop.start() await advance_time(0) # kick the first run diff --git a/tox.ini b/tox.ini index c958e83..2cdb1a6 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ deps = .[testing] mypy commands = - mypy query_exporter {posargs} + mypy {[base]lint_files} {posargs} [testenv:coverage] deps = From b6d52cf1e82c3717d8175cb6185366fdb9da29e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 10:21:37 +0100 Subject: [PATCH 077/110] Bump pygments from 2.18.0 to 2.19.0 in the minor group (#219) Bumps the minor group with 1 update: [pygments](https://github.com/pygments/pygments). Updates `pygments` from 2.18.0 to 2.19.0 - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.18.0...2.19.0) --- updated-dependencies: - dependency-name: pygments dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 72af8c7..bbdd9d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ propcache==0.2.1 # via # aiohttp # yarl -pygments==2.18.0 +pygments==2.19.0 # via rich pytest==8.3.4 # via toolrack From 1dd563e3196a6c6e59afc071f11c14f734b16fd9 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Mon, 6 Jan 2025 20:25:57 +0100 Subject: [PATCH 078/110] Version 3.1.0 --- CHANGES.rst | 11 +++++++++++ pyproject.toml | 9 ++++++--- query_exporter/__init__.py | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index eb79868..82d135f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,14 @@ +v3.1.0 - 2025-01-08 +=================== + +- Support passing multiple configuration files (#108). +- Support YAML tags for ``env``, ``file``, and ``include`` (#188). + +**NOTE**: + The ``env:`` and ``file:`` prefixes for DSNs string form is now deprecated in + favor of the corresponding tags, and will be dropped in the next major release. + + v3.0.0 - 2024-12-30 =================== diff --git a/pyproject.toml b/pyproject.toml index eb0cfa5..d10b9d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,9 +57,12 @@ optional-dependencies.testing = [ "pytest-mock", "pytest-structlog", ] -urls.changelog = "https://github.com/albertodonato/query-exporter/blob/main/CHANGES.rst" -urls.homepage = "https://github.com/albertodonato/query-exporter" -urls.repository = "https://github.com/albertodonato/query-exporter" +urls.Changelog = "https://github.com/albertodonato/query-exporter/blob/main/CHANGES.rst" +urls.Documentation = "https://github.com/albertodonato/query-exporter" +urls.Homepage = "https://github.com/albertodonato/query-exporter" +urls."Issue Tracker" = "https://github.com/albertodonato/query-exporter/issues" +urls."Release Notes" = "https://github.com/albertodonato/query-exporter/blob/main/CHANGES.rst" +urls."Source Code" = "https://github.com/albertodonato/query-exporter" scripts.query-exporter = "query_exporter.main:script" [tool.setuptools.dynamic] diff --git a/query_exporter/__init__.py b/query_exporter/__init__.py index 6bc2e29..cefcbbc 100644 --- a/query_exporter/__init__.py +++ b/query_exporter/__init__.py @@ -1,3 +1,3 @@ """Export Prometheus metrics generated from SQL queries.""" -__version__ = "3.0.0" +__version__ = "3.1.0" From d5bc5b960acb5507116e220a149c8a36a73cb35f Mon Sep 17 00:00:00 2001 From: Lucas Fernando Cardoso Nunes Date: Thu, 9 Jan 2025 04:56:55 -0300 Subject: [PATCH 079/110] typo in README.rst (#220) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a08a812..5ac8d9f 100644 --- a/README.rst +++ b/README.rst @@ -116,7 +116,7 @@ switches, environment variables or through the ``.env`` file: | ``--metrics-path`` | ``QE_METRICS_PATH`` | ``/metrics`` | Path under which metrics are exposed. | +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ | ``-L``, ``--log-level`` | ``QE_LOG_LEVEL`` | ``info`` | Minimum level for log messages level. | - | | | | One of ``critical``, ``error``, ``wanring``, ``info``, ``debug``. | + | | | | One of ``critical``, ``error``, ``warning``, ``info``, ``debug``. | +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ | ``--log-format`` | ``QE_LOG_FORMAT`` | ``plain`` | Log output format. One of ``plain``, ``json``. | +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ From 6003044dd194df687f3ab355bd51566b89879930 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 9 Jan 2025 13:17:40 +0100 Subject: [PATCH 080/110] chore: add PyPI downloads badge to readme --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5ac8d9f..924a9f0 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ Export Prometheus metrics from SQL queries ========================================== -|Latest Version| |Build Status| |Snap Package| |Docker Pulls| +|Latest Version| |Build Status| |PyPI Downloads| |Docker Pulls| |Snap Package| ``query-exporter`` is a Prometheus_ exporter which allows collecting metrics from database queries, at specified time intervals. @@ -285,3 +285,6 @@ The snap has support for connecting the following databases: .. |Docker Pulls| image:: https://img.shields.io/docker/pulls/adonato/query-exporter :alt: Docker Pulls :target: https://hub.docker.com/r/adonato/query-exporter +.. |PyPI Downloads| + :alt: PyPI Downloads + :target: https://static.pepy.tech/badge/query-exporter/month From 726fefb315d4b47c961fdff2d4cab291435f108a Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 9 Jan 2025 13:18:04 +0100 Subject: [PATCH 081/110] chore: extract module with builtin metrics --- query_exporter/config.py | 65 +++------------------------------------ query_exporter/loop.py | 10 +++--- query_exporter/metrics.py | 65 +++++++++++++++++++++++++++++++++++++++ tests/config_test.py | 20 ++++++------ 4 files changed, 86 insertions(+), 74 deletions(-) create mode 100644 query_exporter/metrics.py diff --git a/query_exporter/config.py b/query_exporter/config.py index a76b377..4523e6c 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -28,51 +28,9 @@ Query, QueryMetric, ) +from .metrics import BUILTIN_METRICS, get_builtin_metric_configs from .yaml import load_yaml_config -# metric for counting database errors -DB_ERRORS_METRIC_NAME = "database_errors" -_DB_ERRORS_METRIC_CONFIG = MetricConfig( - name=DB_ERRORS_METRIC_NAME, - description="Number of database errors", - type="counter", - config={"increment": True}, -) - -# metric for counting performed queries -QUERIES_METRIC_NAME = "queries" -_QUERIES_METRIC_CONFIG = MetricConfig( - name=QUERIES_METRIC_NAME, - description="Number of database queries", - type="counter", - labels=("query", "status"), - config={"increment": True}, -) -# metric for tracking last query execution timestamp -QUERY_TIMESTAMP_METRIC_NAME = "query_timestamp" -_QUERY_TIMESTAMP_METRIC_CONFIG = MetricConfig( - name=QUERY_TIMESTAMP_METRIC_NAME, - description="Query last execution timestamp", - type="gauge", - labels=("query",), -) -# metric for counting queries execution latency -QUERY_LATENCY_METRIC_NAME = "query_latency" -_QUERY_LATENCY_METRIC_CONFIG = MetricConfig( - name=QUERY_LATENCY_METRIC_NAME, - description="Query execution latency", - type="histogram", - labels=("query",), -) -GLOBAL_METRICS = frozenset( - ( - DB_ERRORS_METRIC_NAME, - QUERIES_METRIC_NAME, - QUERY_LATENCY_METRIC_NAME, - QUERY_TIMESTAMP_METRIC_NAME, - ) -) - # regexp for validating environment variables names _ENV_VAR_RE = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$") @@ -189,22 +147,7 @@ def _get_metrics( metrics: dict[str, dict[str, t.Any]], extra_labels: frozenset[str] ) -> dict[str, MetricConfig]: """Return a dict mapping metric names to their configuration.""" - configs = {} - # global metrics - for metric_config in ( - _DB_ERRORS_METRIC_CONFIG, - _QUERIES_METRIC_CONFIG, - _QUERY_LATENCY_METRIC_CONFIG, - _QUERY_TIMESTAMP_METRIC_CONFIG, - ): - configs[metric_config.name] = MetricConfig( - metric_config.name, - metric_config.description, - metric_config.type, - labels=set(metric_config.labels) | extra_labels, - config=metric_config.config, - ) - # other metrics + configs = get_builtin_metric_configs(extra_labels) for name, config in metrics.items(): _validate_metric_config(name, config, extra_labels) metric_type = config.pop("type") @@ -221,7 +164,7 @@ def _validate_metric_config( name: str, config: dict[str, t.Any], extra_labels: frozenset[str] ) -> None: """Validate a metric configuration stanza.""" - if name in GLOBAL_METRICS: + if name in BUILTIN_METRICS: raise ConfigError(f'Label name "{name} is reserved for builtin metric') labels = set(config.get("labels", ())) overlap_labels = labels & extra_labels @@ -437,7 +380,7 @@ def _warn_if_unused( entries=unused_dbs, ) if unused_metrics := sorted( - set(config.metrics) - GLOBAL_METRICS - used_metrics + set(config.metrics) - BUILTIN_METRICS - used_metrics ): logger.warning( "unused config entries", diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 839d86a..c25f7fd 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -23,10 +23,6 @@ ) from .config import ( - DB_ERRORS_METRIC_NAME, - QUERIES_METRIC_NAME, - QUERY_LATENCY_METRIC_NAME, - QUERY_TIMESTAMP_METRIC_NAME, Config, ) from .db import ( @@ -37,6 +33,12 @@ Query, QueryTimeoutExpired, ) +from .metrics import ( + DB_ERRORS_METRIC_NAME, + QUERIES_METRIC_NAME, + QUERY_LATENCY_METRIC_NAME, + QUERY_TIMESTAMP_METRIC_NAME, +) class MetricsLastSeen: diff --git a/query_exporter/metrics.py b/query_exporter/metrics.py new file mode 100644 index 0000000..45cf620 --- /dev/null +++ b/query_exporter/metrics.py @@ -0,0 +1,65 @@ +from prometheus_aioexporter import MetricConfig + +# metric for counting database errors +DB_ERRORS_METRIC_NAME = "database_errors" +_DB_ERRORS_METRIC_CONFIG = MetricConfig( + name=DB_ERRORS_METRIC_NAME, + description="Number of database errors", + type="counter", + config={"increment": True}, +) + +# metric for counting performed queries +QUERIES_METRIC_NAME = "queries" +_QUERIES_METRIC_CONFIG = MetricConfig( + name=QUERIES_METRIC_NAME, + description="Number of database queries", + type="counter", + labels=("query", "status"), + config={"increment": True}, +) +# metric for tracking last query execution timestamp +QUERY_TIMESTAMP_METRIC_NAME = "query_timestamp" +_QUERY_TIMESTAMP_METRIC_CONFIG = MetricConfig( + name=QUERY_TIMESTAMP_METRIC_NAME, + description="Query last execution timestamp", + type="gauge", + labels=("query",), +) +# metric for counting queries execution latency +QUERY_LATENCY_METRIC_NAME = "query_latency" +_QUERY_LATENCY_METRIC_CONFIG = MetricConfig( + name=QUERY_LATENCY_METRIC_NAME, + description="Query execution latency", + type="histogram", + labels=("query",), +) +BUILTIN_METRICS = frozenset( + ( + DB_ERRORS_METRIC_NAME, + QUERIES_METRIC_NAME, + QUERY_LATENCY_METRIC_NAME, + QUERY_TIMESTAMP_METRIC_NAME, + ) +) + + +def get_builtin_metric_configs( + extra_labels: frozenset[str], +) -> dict[str, MetricConfig]: + """Return configuration for builtin metrics.""" + return { + metric_config.name: MetricConfig( + metric_config.name, + metric_config.description, + metric_config.type, + labels=set(metric_config.labels) | extra_labels, + config=metric_config.config, + ) + for metric_config in ( + _DB_ERRORS_METRIC_CONFIG, + _QUERIES_METRIC_CONFIG, + _QUERY_LATENCY_METRIC_CONFIG, + _QUERY_TIMESTAMP_METRIC_CONFIG, + ) + } diff --git a/tests/config_test.py b/tests/config_test.py index cd57a5a..b73e8da 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -8,15 +8,17 @@ import yaml from query_exporter.config import ( - DB_ERRORS_METRIC_NAME, - GLOBAL_METRICS, - QUERIES_METRIC_NAME, ConfigError, _get_parameters_sets, _resolve_dsn, load_config, ) from query_exporter.db import QueryMetric +from query_exporter.metrics import ( + BUILTIN_METRICS, + DB_ERRORS_METRIC_NAME, + QUERIES_METRIC_NAME, +) @pytest.fixture @@ -437,7 +439,7 @@ def test_load_metrics_section(self, write_config: ConfigWriter) -> None: "states": ["on", "off"], "expiration": 100, } - # global metrics + # builtin metrics assert result.metrics.get(DB_ERRORS_METRIC_NAME) is not None assert result.metrics.get(QUERIES_METRIC_NAME) is not None @@ -473,20 +475,20 @@ def test_load_metrics_overlap_database_label( == 'Labels for metric "m" overlap with reserved/database ones: l1' ) - @pytest.mark.parametrize("global_name", list(GLOBAL_METRICS)) + @pytest.mark.parametrize("builtin_metric_name", list(BUILTIN_METRICS)) def test_load_metrics_reserved_name( self, config_full: dict[str, t.Any], write_config: ConfigWriter, - global_name: str, + builtin_metric_name: str, ) -> None: - config_full["metrics"][global_name] = {"type": "counter"} + config_full["metrics"][builtin_metric_name] = {"type": "counter"} config_file = write_config(config_full) with pytest.raises(ConfigError) as err: load_config([config_file]) assert ( str(err.value) - == f'Label name "{global_name} is reserved for builtin metric' + == f'Label name "{builtin_metric_name} is reserved for builtin metric' ) def test_load_metrics_unsupported_type( @@ -1006,7 +1008,7 @@ def test_load_multiple_files_combine( ) config = load_config([file1, file2]) assert set(config.databases) == {"db1", "db2"} - assert set(config.metrics) == {"m1", "m2"} | GLOBAL_METRICS + assert set(config.metrics) == {"m1", "m2"} | BUILTIN_METRICS assert set(config.queries) == {"q1", "q2"} def test_load_multiple_files_duplicated_database( From 4c79e93560f76ecffa6d3a384bdb828d16b69c6c Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 9 Jan 2025 18:12:42 +0100 Subject: [PATCH 082/110] chore: fix badge in README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 924a9f0..785188c 100644 --- a/README.rst +++ b/README.rst @@ -285,6 +285,6 @@ The snap has support for connecting the following databases: .. |Docker Pulls| image:: https://img.shields.io/docker/pulls/adonato/query-exporter :alt: Docker Pulls :target: https://hub.docker.com/r/adonato/query-exporter -.. |PyPI Downloads| +.. |PyPI Downloads| image:: https://static.pepy.tech/badge/query-exporter/month :alt: PyPI Downloads - :target: https://static.pepy.tech/badge/query-exporter/month + :target: https://pepy.tech/projects/query-exporter From 1bc41390a16713c08683ebbf05bbbef0efa0f97e Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 11 Jan 2025 21:38:40 +0100 Subject: [PATCH 083/110] chore: drop unused s390x arch from packages --- Dockerfile | 2 +- snap/snapcraft.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 91391cb..413acbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN pip install \ /srcdir \ cx-Oracle \ clickhouse-sqlalchemy \ - "ibm-db-sa; platform_machine == 'x86_64' or platform_machine == 'ppc64le' or platform_machine == 's390x'" \ + "ibm-db-sa; platform_machine == 'x86_64' or platform_machine == 'ppc64le'" \ mysqlclient \ psycopg2 \ pymssql \ diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 1900c9f..3fd55e2 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -68,7 +68,6 @@ parts: - clickhouse-sqlalchemy - ibm-db-sa; platform_machine == 'x86_64' - ibm-db-sa; platform_machine == 'ppc64le' - - ibm-db-sa; platform_machine == 's390x' - mysqlclient - psycopg2 - pyodbc From 6faf55b16f349346bc5419e0bcdf04250f244e75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 11:04:53 +0100 Subject: [PATCH 084/110] Bump sqlalchemy from 2.0.36 to 2.0.37 (#221) Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 2.0.36 to 2.0.37. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bbdd9d8..da0c343 100644 --- a/requirements.txt +++ b/requirements.txt @@ -83,7 +83,7 @@ rpds-py==0.22.3 # referencing six==1.17.0 # via python-dateutil -sqlalchemy==2.0.36 +sqlalchemy==2.0.37 # via query-exporter (pyproject.toml) structlog==24.4.0 # via From 37d8be95b0299d168f9d6b968cae99d0490ad8c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 11:05:34 +0100 Subject: [PATCH 085/110] Bump pygments from 2.19.0 to 2.19.1 (#222) Bumps [pygments](https://github.com/pygments/pygments) from 2.19.0 to 2.19.1. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.19.0...2.19.1) --- updated-dependencies: - dependency-name: pygments dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index da0c343..bd83398 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ propcache==0.2.1 # via # aiohttp # yarl -pygments==2.19.0 +pygments==2.19.1 # via rich pytest==8.3.4 # via toolrack From 672ebfef76a54178437924bd318309eccdfa0d28 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Thu, 16 Jan 2025 13:21:49 +0100 Subject: [PATCH 086/110] chore: combine execute logic --- query_exporter/db.py | 12 ++++++------ tests/db_test.py | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/query_exporter/db.py b/query_exporter/db.py index e82c1f5..c5727ba 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -345,11 +345,10 @@ async def execute( """Execute a query, returning results.""" if parameters is None: parameters = {} - result = await self._call_in_thread(self._execute, sql, parameters) - query_results: QueryResults = await self._call_in_thread( - QueryResults.from_result, result + return t.cast( + QueryResults, + await self._call_in_thread(self._execute, sql, parameters), ) - return query_results def _create_worker(self) -> None: assert not self._worker @@ -368,9 +367,10 @@ def _connect(self) -> None: def _execute( self, sql: TextClause, parameters: dict[str, t.Any] - ) -> CursorResult[t.Any]: + ) -> QueryResults: assert self._conn - return self._conn.execute(sql, parameters) + result = self._conn.execute(sql, parameters) + return QueryResults.from_result(result) def _close(self) -> None: assert self._conn diff --git a/tests/db_test.py b/tests/db_test.py index 45d3278..c7ac05d 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -502,7 +502,6 @@ async def test_execute_log( await db.execute(query) assert log.has("run query", query="query", database="db") assert log.has("action received", worker_id=ANY, action="_execute") - assert log.has("action received", worker_id=ANY, action="from_result") await db.close() @pytest.mark.parametrize("connected", [True, False]) From 24c37179cbba6f353829bfdba00d1d890d7f5187 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Fri, 17 Jan 2025 18:02:06 +0100 Subject: [PATCH 087/110] chore: move label definition --- query_exporter/config.py | 4 +++- query_exporter/db.py | 6 +----- query_exporter/loop.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/query_exporter/config.py b/query_exporter/config.py index 4523e6c..67b508a 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -21,7 +21,6 @@ import yaml from .db import ( - DATABASE_LABEL, DataBaseConfig, InvalidQueryParameters, InvalidQuerySchedule, @@ -31,6 +30,9 @@ from .metrics import BUILTIN_METRICS, get_builtin_metric_configs from .yaml import load_yaml_config +# Label used to tag metrics by database +DATABASE_LABEL = "database" + # regexp for validating environment variables names _ENV_VAR_RE = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$") diff --git a/query_exporter/db.py b/query_exporter/db.py index c5727ba..52c0f40 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -41,14 +41,10 @@ from sqlalchemy.sql.elements import TextClause import structlog -#: Timeout for a query +# Timeout for a query QueryTimeout = int | float -#: Label used to tag metrics by database -DATABASE_LABEL = "database" - - class DataBaseError(Exception): """A databease error. diff --git a/query_exporter/loop.py b/query_exporter/loop.py index c25f7fd..4f6b9ad 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -23,10 +23,10 @@ ) from .config import ( + DATABASE_LABEL, Config, ) from .db import ( - DATABASE_LABEL, DataBase, DataBaseConnectError, DataBaseError, From 9602756e7ef2bd7a6a42d22e2c4ea25027d45bd5 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Sat, 18 Jan 2025 10:56:05 +0100 Subject: [PATCH 088/110] chore: add contribution guidelines (#226) --- README.rst | 17 +++++++--- docs/contributing.rst | 78 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 docs/contributing.rst diff --git a/README.rst b/README.rst index 785188c..4c78f76 100644 --- a/README.rst +++ b/README.rst @@ -132,12 +132,11 @@ switches, environment variables or through the ``.env`` file: +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ | ``--config`` | ``QE_CONFIG`` | ``config.yaml`` | Configuration files. Multiple values can be provided. | +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ - | | ``QE_DOTENV`` | ``$PWD/.env`` | Path for the dotenv file where environment variables can be | + | | ``QE_DOTENV`` | ``$PWD/.env`` | Path for the dotenv file where environment variables can be | | | | | provided. | +-------------------------+------------------------+-----------------+-------------------------------------------------------------------+ - Metrics endpoint ---------------- @@ -184,7 +183,7 @@ See `supported databases`_ for details. Run in Docker -------------- +============= ``query-exporter`` can be run inside Docker_ containers, and is available from the `Docker Hub`_:: @@ -218,7 +217,7 @@ Automated builds from the ``main`` branch are available on the `GitHub container ODBC driver version -~~~~~~~~~~~~~~~~~~~ +------------------- A different ODBC driver version to use can be specified during image building, by passing ``--build-arg ODBC_bVERSION_NUMBER``, e.g.:: @@ -227,7 +226,7 @@ by passing ``--build-arg ODBC_bVERSION_NUMBER``, e.g.:: Install from Snap ------------------ +================= |Get it from the Snap Store| @@ -257,6 +256,13 @@ The snap has support for connecting the following databases: s390x) +Contributing +============ + +The project welcomes contributions of any form. Please refer to the +`contribution guide`_ for details on how to contribute. + + .. _Prometheus: https://prometheus.io/ .. _SQLAlchemy: https://www.sqlalchemy.org/ .. _`supported databases`: @@ -265,6 +271,7 @@ The snap has support for connecting the following databases: .. _Docker: http://docker.com/ .. _`Docker Hub`: https://hub.docker.com/r/adonato/query-exporter .. _`configuration file format`: docs/configuration.rst +.. _`contribution guide`: docs/contributing.rst .. _`Helm chart`: https://github.com/makezbs/helm-charts/tree/main/charts/query-exporter .. _`GitHub container registry`: https://github.com/albertodonato/query-exporter/pkgs/container/query-exporter diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..28d1870 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,78 @@ +Thanks for considering contributing to query-exporter! + +Issues +====== + +When submitting an issue, please include a detailed description of the issue, +what is happening, and in which conditions. + +If possible, attach a log of the exporter with debug enabled (``--log-level +debug``), as well as the (sanitized as needed) content of the configuration +file. + +Always include the exporter version (``query-exporter --version``), as well as +the installation method. + + +Enhancements +============ + +When proposing enhancements, please describe in detail the use cases or +problems that the enhancement would solve. + +If possible, include examples of the new behavior with the change. + + +Pull requests +============= + +Creating pull requests is very easy, and requires just a minimal development +setup is requested to verify the changes. + +When creating a pull request for a non-trivial bug or enhancement, please +consider creating an issue first, so that discussion can happen more easily, +and reference it in the pull request. + +Prerequisites +------------- + +The development environment requires having ``tox`` installed. Please refer to +to the `Tox wiki`_ for installation instructions. + +Please make sure that you run the following steps on your changes before +creating the pull request. + +Tests +----- + +Changes must have full test coverage. The full suite can be run via:: + + tox run -e coverage + +which will include the coverage report. + +To just run the tests, possibly limiting to a subset of them, run:: + + tox run -e py -- + +Type checking +------------- + +The project uses ``mypy`` for type checking. Please make sure types are added correctly to new/changed code. +To verify, run:: + + tox run -e check + +Linting and formatting +---------------------- + +Formatting can be applied automatically with:: + + tox run -e format + +Linting is checked on pull requests, and can be verified with:: + + tox run -e lint + + +.. _`Tox wiki`: https://tox.wiki/en/latest/index.html From c5880c29da3829376ef7edf5e6c03cc77aa3e373 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:34:07 +0100 Subject: [PATCH 089/110] Bump referencing from 0.35.1 to 0.36.1 in the minor group (#227) Bumps the minor group with 1 update: [referencing](https://github.com/python-jsonschema/referencing). Updates `referencing` from 0.35.1 to 0.36.1 - [Release notes](https://github.com/python-jsonschema/referencing/releases) - [Changelog](https://github.com/python-jsonschema/referencing/blob/main/docs/changes.rst) - [Commits](https://github.com/python-jsonschema/referencing/compare/v0.35.1...v0.36.1) --- updated-dependencies: - dependency-name: referencing dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bd83398..373b1ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -71,7 +71,7 @@ pytz==2024.2 # via croniter pyyaml==6.0.2 # via query-exporter (pyproject.toml) -referencing==0.35.1 +referencing==0.36.1 # via # jsonschema # jsonschema-specifications From 018fb18cfdad9c51f3e03fa185a639d8796bb33b Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 21 Jan 2025 17:13:13 +0100 Subject: [PATCH 090/110] chore: refactory Query adding QueryExecution (#229) --- query_exporter/config.py | 43 +++++++-------------- query_exporter/db.py | 49 ++++++++++++++++++------ query_exporter/loop.py | 76 ++++++++++++++++++++++--------------- tests/config_test.py | 81 +++++++++++++++++----------------------- tests/conftest.py | 12 +++--- tests/db_test.py | 66 ++++++++++++++++++-------------- 6 files changed, 177 insertions(+), 150 deletions(-) diff --git a/query_exporter/config.py b/query_exporter/config.py index 67b508a..cf5e620 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -189,34 +189,17 @@ def _get_queries( for name, config in configs.items(): _validate_query_config(name, config, database_names, metric_names) query_metrics = _get_query_metrics(config, metrics, extra_labels) - parameters = config.get("parameters") - - query_args = { - "databases": config["databases"], - "metrics": query_metrics, - "sql": config["sql"].strip(), - "timeout": config.get("timeout"), - "interval": _convert_interval(config.get("interval")), - "schedule": config.get("schedule"), - "config_name": name, - } - try: - if parameters: - parameters_sets = _get_parameters_sets(parameters) - queries.update( - ( - f"{name}[params{index}]", - Query( - name=f"{name}[params{index}]", - parameters=params, - **query_args, - ), - ) - for index, params in enumerate(parameters_sets) - ) - else: - queries[name] = Query(name, **query_args) + queries[name] = Query( + name=name, + databases=config["databases"], + metrics=query_metrics, + sql=config["sql"].strip(), + timeout=config.get("timeout"), + interval=_convert_interval(config.get("interval")), + schedule=config.get("schedule"), + parameter_sets=_get_parameter_sets(config.get("parameters")), + ) except (InvalidQueryParameters, InvalidQuerySchedule) as e: raise ConfigError(str(e)) return queries @@ -391,10 +374,12 @@ def _warn_if_unused( ) -def _get_parameters_sets( - parameters: ParametersConfig, +def _get_parameter_sets( + parameters: ParametersConfig | None, ) -> list[dict[str, t.Any]]: """Return an sequence of set of paramters with their values.""" + if not parameters: + return [] if isinstance(parameters, dict): return _get_parameters_matrix(parameters) return parameters diff --git a/query_exporter/db.py b/query_exporter/db.py index 52c0f40..3d764ff 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -7,6 +7,7 @@ Sequence, ) from dataclasses import ( + InitVar, dataclass, field, ) @@ -194,17 +195,24 @@ class Query: databases: list[str] metrics: list[QueryMetric] sql: str - parameters: dict[str, t.Any] = field(default_factory=dict) timeout: QueryTimeout | None = None interval: int | None = None schedule: str | None = None - config_name: str = "" - def __post_init__(self) -> None: - if not self.config_name: - self.config_name = self.name + parameter_sets: InitVar[list[dict[str, t.Any]] | None] = None + executions: list["QueryExecution"] = field(init=False, compare=False) + + def __post_init__( + self, parameter_sets: list[dict[str, t.Any]] | None + ) -> None: self._check_schedule() - self._check_query_parameters() + if not parameter_sets: + self.executions = [QueryExecution(self.name, self)] + else: + self.executions = [ + QueryExecution(f"{self.name}[params{index}]", self, parameters) + for index, parameters in enumerate(parameter_sets, 1) + ] @property def timed(self) -> bool: @@ -253,8 +261,20 @@ def _check_schedule(self) -> None: if self.schedule and not croniter.is_valid(self.schedule): raise InvalidQuerySchedule(self.name, "invalid schedule format") + +@dataclass(frozen=True) +class QueryExecution: + """A single execution configuration for a query, with parameters.""" + + name: str + query: Query + parameters: dict[str, t.Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + self._check_query_parameters() + def _check_query_parameters(self) -> None: - expr = text(self.sql) + expr = text(self.query.sql) query_params = set(expr.compile().params) if set(self.parameters) != query_params: raise InvalidQueryParameters(self.name) @@ -472,22 +492,27 @@ async def close(self) -> None: return await self._close() - async def execute(self, query: Query) -> MetricResults: + async def execute(self, query_execution: QueryExecution) -> MetricResults: """Execute a query.""" await self.connect() - self.logger.debug("run query", query=query.name) + self.logger.debug("run query", query=query_execution.name) self._pending_queries += 1 + query = query_execution.query try: query_results = await self.execute_sql( - query.sql, parameters=query.parameters, timeout=query.timeout + query.sql, + parameters=query_execution.parameters, + timeout=query.timeout, ) return query.results(query_results) except TimeoutError: - self.logger.warning("query timeout", query=query.name) + self.logger.warning("query timeout", query=query_execution.name) raise QueryTimeoutExpired() except Exception as error: raise self._query_db_error( - query.name, error, fatal=isinstance(error, FATAL_ERRORS) + query_execution.name, + error, + fatal=isinstance(error, FATAL_ERRORS), ) finally: assert self._pending_queries >= 0, "pending queries is negative" diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 4f6b9ad..08f584c 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -5,6 +5,7 @@ from collections.abc import Iterator, Mapping from datetime import datetime from decimal import Decimal +from itertools import chain import time import typing as t @@ -31,6 +32,7 @@ DataBaseConnectError, DataBaseError, Query, + QueryExecution, QueryTimeoutExpired, ) from .metrics import ( @@ -111,8 +113,8 @@ def __init__( self._config = config self._registry = registry self._logger = logger or structlog.get_logger() - self._timed_queries: list[Query] = [] - self._aperiodic_queries: list[Query] = [] + self._timed_query_executions: list[QueryExecution] = [] + self._aperiodic_query_executions: list[QueryExecution] = [] # map query names to their TimedCalls self._timed_calls: dict[str, TimedCall] = {} # map query names to list of database names @@ -129,23 +131,26 @@ def __init__( for db_config in self._config.databases.values() } - for query in self._config.queries.values(): - if query.timed: - self._timed_queries.append(query) + for query_execution in chain( + *(query.executions for query in self._config.queries.values()) + ): + if query_execution.query.timed: + self._timed_query_executions.append(query_execution) else: - self._aperiodic_queries.append(query) + self._aperiodic_query_executions.append(query_execution) async def start(self) -> None: """Start timed queries execution.""" - for query in self._timed_queries: + for query_execution in self._timed_query_executions: + query = query_execution.query call: TimedCall if query.interval: - call = PeriodicCall(self._run_query, query) + call = PeriodicCall(self._run_query, query_execution) call.start(query.interval, now=True) elif query.schedule is not None: - call = TimedCall(self._run_query, query) + call = TimedCall(self._run_query, query_execution) call.start(self._loop_times_iter(query.schedule)) - self._timed_calls[query.name] = call + self._timed_calls[query_execution.name] = call async def stop(self) -> None: """Stop timed query execution.""" @@ -166,9 +171,9 @@ def clear_expired_series(self) -> None: async def run_aperiodic_queries(self) -> None: """Run queries on request.""" coros = ( - self._execute_query(query, dbname) - for query in self._aperiodic_queries - for dbname in query.databases + self._execute_query(query_execution, dbname) + for query_execution in self._aperiodic_query_executions + for dbname in query_execution.query.databases ) await asyncio.gather(*coros, return_exceptions=True) @@ -181,19 +186,24 @@ def _loop_times_iter(self, schedule: str) -> Iterator[float | int]: delta = cc - t yield self._loop.time() + delta - def _run_query(self, query: Query) -> None: + def _run_query(self, query_execution: QueryExecution) -> None: """Periodic task to run a query.""" - for dbname in query.databases: - self._loop.create_task(self._execute_query(query, dbname)) + for dbname in query_execution.query.databases: + self._loop.create_task( + self._execute_query(query_execution, dbname) + ) - async def _execute_query(self, query: Query, dbname: str) -> None: + async def _execute_query( + self, query_execution: QueryExecution, dbname: str + ) -> None: """'Execute a Query on a DataBase.""" - if await self._remove_if_dooomed(query, dbname): + if await self._remove_if_dooomed(query_execution, dbname): return db = self._databases[dbname] + query = query_execution.query try: - metric_results = await db.execute(query) + metric_results = await db.execute(query_execution) except DataBaseConnectError: self._increment_db_error_count(db) except QueryTimeoutExpired: @@ -203,10 +213,10 @@ async def _execute_query(self, query: Query, dbname: str) -> None: if error.fatal: self._logger.debug( "removing failed query", - query=query.name, + query=query_execution.name, database=dbname, ) - self._doomed_queries[query.name].add(dbname) + self._doomed_queries[query_execution.name].add(dbname) else: for result in metric_results.results: self._update_metric( @@ -222,24 +232,28 @@ async def _execute_query(self, query: Query, dbname: str) -> None: ) self._increment_queries_count(db, query, "success") - async def _remove_if_dooomed(self, query: Query, dbname: str) -> bool: - """Remove a query if it will never work. + async def _remove_if_dooomed( + self, query_execution: QueryExecution, dbname: str + ) -> bool: + """Remove a query execution if it will never work. Return whether the query has been removed for the database. """ - if dbname not in self._doomed_queries[query.name]: + if dbname not in self._doomed_queries[query_execution.name]: return False - if set(query.databases) == self._doomed_queries[query.name]: + query = query_execution.query + + if set(query.databases) == self._doomed_queries[query_execution.name]: # the query has failed on all databases if query.timed: - self._timed_queries.remove(query) - call = self._timed_calls.pop(query.name, None) + self._timed_query_executions.remove(query_execution) + call = self._timed_calls.pop(query_execution.name, None) if call is not None: await call.stop() else: - self._aperiodic_queries.remove(query) + self._aperiodic_query_executions.remove(query_execution) return True def _update_metric( @@ -304,7 +318,7 @@ def _increment_queries_count( database, QUERIES_METRIC_NAME, 1, - labels={"query": query.config_name, "status": status}, + labels={"query": query.name, "status": status}, ) def _increment_db_error_count(self, database: DataBase) -> None: @@ -319,7 +333,7 @@ def _update_query_latency_metric( database, QUERY_LATENCY_METRIC_NAME, latency, - labels={"query": query.config_name}, + labels={"query": query.name}, ) def _update_query_timestamp_metric( @@ -330,5 +344,5 @@ def _update_query_timestamp_metric( database, QUERY_TIMESTAMP_METRIC_NAME, timestamp, - labels={"query": query.config_name}, + labels={"query": query.name}, ) diff --git a/tests/config_test.py b/tests/config_test.py index b73e8da..567bc39 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -9,7 +9,7 @@ from query_exporter.config import ( ConfigError, - _get_parameters_sets, + _get_parameter_sets, _resolve_dsn, load_config, ) @@ -536,18 +536,19 @@ def test_load_queries_section(self, write_config: ConfigWriter) -> None: } config_file = write_config(cfg) result = load_config([config_file]) + assert len(result.queries) == 2 query1 = result.queries["q1"] assert query1.name == "q1" assert query1.databases == ["db1"] assert query1.metrics == [QueryMetric("m1", ["l1", "l2"])] assert query1.sql == "SELECT 1" - assert query1.parameters == {} + assert len(query1.executions) == 1 query2 = result.queries["q2"] assert query2.name == "q2" assert query2.databases == ["db2"] assert query2.metrics == [QueryMetric("m2", [])] assert query2.sql == "SELECT 2" - assert query2.parameters == {} + assert len(query2.executions) == 1 def test_load_queries_section_with_parameters( self, write_config: ConfigWriter @@ -570,21 +571,20 @@ def test_load_queries_section_with_parameters( } config_file = write_config(cfg) result = load_config([config_file]) - query1 = result.queries["q[params0]"] - assert query1.name == "q[params0]" - assert query1.databases == ["db"] - assert query1.metrics == [QueryMetric("m", ["l"])] - assert query1.sql == "SELECT :param1 AS l, :param2 AS m" - assert query1.parameters == { + assert len(result.queries) == 1 + query = result.queries["q"] + assert query.name == "q" + assert query.databases == ["db"] + assert query.metrics == [QueryMetric("m", ["l"])] + assert query.sql == "SELECT :param1 AS l, :param2 AS m" + query_exec1, query_exec2 = query.executions + assert query_exec1.name == "q[params1]" + assert query_exec1.parameters == { "param1": "label1", "param2": 10, } - query2 = result.queries["q[params1]"] - assert query2.name == "q[params1]" - assert query2.databases == ["db"] - assert query2.metrics == [QueryMetric("m", ["l"])] - assert query2.sql == "SELECT :param1 AS l, :param2 AS m" - assert query2.parameters == { + assert query_exec2.name == "q[params2]" + assert query_exec2.parameters == { "param1": "label2", "param2": 20, } @@ -610,43 +610,32 @@ def test_load_queries_section_with_parameters_matrix( } config_file = write_config(cfg) result = load_config([config_file]) + assert len(result.queries) == 1 + query = result.queries["q"] + assert query.databases == ["db"] + assert query.metrics == [QueryMetric("m", ["l"])] + assert ( + query.sql == "SELECT :marketplace__name AS l, :item__status AS m" + ) - assert len(result.queries) == 4 - - # check common props for each query - for query_name, query in result.queries.items(): - assert query.databases == ["db"] - assert query.metrics == [QueryMetric("m", ["l"])] - assert ( - query.sql - == "SELECT :marketplace__name AS l, :item__status AS m" - ) - - # Q1 - query1 = result.queries["q[params0]"] - assert query1.name == "q[params0]" - assert query1.parameters == { + query_exec1, query_exec2, query_exec3, query_exec4 = query.executions + assert query_exec1.name == "q[params1]" + assert query_exec1.parameters == { "marketplace__name": "amazon", "item__status": "active", } - # Q2 - query2 = result.queries["q[params1]"] - assert query2.name == "q[params1]" - assert query2.parameters == { + assert query_exec2.name == "q[params2]" + assert query_exec2.parameters == { "marketplace__name": "ebay", "item__status": "active", } - # Q3 - query3 = result.queries["q[params2]"] - assert query3.name == "q[params2]" - assert query3.parameters == { + assert query_exec3.name == "q[params3]" + assert query_exec3.parameters == { "marketplace__name": "amazon", "item__status": "inactive", } - # Q4 - query4 = result.queries["q[params3]"] - assert query4.name == "q[params3]" - assert query4.parameters == { + assert query_exec4.name == "q[params4]" + assert query_exec4.parameters == { "marketplace__name": "ebay", "item__status": "inactive", } @@ -675,7 +664,7 @@ def test_load_queries_section_with_wrong_parameters( load_config([config_file]) assert ( str(err.value) - == 'Parameters for query "q[params0]" don\'t match those from SQL' + == 'Parameters for query "q[params1]" don\'t match those from SQL' ) def test_load_queries_section_with_schedule_and_interval( @@ -1103,7 +1092,7 @@ def test_encode_options(self) -> None: ) -class TestGetParametersSets: +class TestGetParameterSets: def test_list(self) -> None: params: list[dict[str, t.Any]] = [ { @@ -1115,7 +1104,7 @@ def test_list(self) -> None: "param2": "bar", }, ] - assert list(_get_parameters_sets(params)) == params + assert list(_get_parameter_sets(params)) == params def test_dict(self) -> None: params: dict[str, list[dict[str, t.Any]]] = { @@ -1148,7 +1137,7 @@ def test_dict(self) -> None: }, ], } - assert list(_get_parameters_sets(params)) == [ + assert list(_get_parameter_sets(params)) == [ { "param1__sub1": 100, "param1__sub2": "foo", diff --git a/tests/conftest.py b/tests/conftest.py index ab0cbfc..46cbd80 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,7 @@ from pytest_structlog import StructuredLogCapture from toolrack.testing.fixtures import advance_time -from query_exporter.db import DataBase, MetricResults, Query +from query_exporter.db import DataBase, MetricResults, QueryExecution __all__ = ["QueryTracker", "advance_time", "query_tracker"] @@ -19,7 +19,7 @@ def _autouse(log: StructuredLogCapture) -> Iterator[None]: class QueryTracker: def __init__(self) -> None: - self.queries: list[Query] = [] + self.queries: list[QueryExecution] = [] self.results: list[MetricResults] = [] self.failures: list[Exception] = [] self._loop = asyncio.get_event_loop() @@ -51,10 +51,12 @@ async def query_tracker( tracker = QueryTracker() orig_execute = DataBase.execute - async def execute(db: DataBase, query: Query) -> MetricResults: - tracker.queries.append(query) + async def execute( + db: DataBase, query_execution: QueryExecution + ) -> MetricResults: + tracker.queries.append(query_execution) try: - result = await orig_execute(db, query) + result = await orig_execute(db, query_execution) except Exception as e: tracker.failures.append(e) raise diff --git a/tests/db_test.py b/tests/db_test.py index c7ac05d..1c3d570 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -70,26 +70,16 @@ def test_instantiate(self) -> None: "SELECT 1", ) assert query.name == "query" - assert query.config_name == "query" assert query.databases == ["db1", "db2"] assert query.metrics == [ QueryMetric("metric1", ["label1", "label2"]), QueryMetric("metric2", ["label2"]), ] assert query.sql == "SELECT 1" - assert query.parameters == {} assert query.interval is None assert query.timeout is None - - def test_instantiate_with_config_name(self) -> None: - query = Query( - "query", - ["db"], - [QueryMetric("metric", [])], - "SELECT metric1 FROM table", - config_name="query_config", - ) - assert query.config_name == "query_config" + [query_execution] = query.executions + assert query_execution.name == "query" def test_instantiate_with_parameters(self) -> None: query = Query( @@ -101,9 +91,16 @@ def test_instantiate_with_parameters(self) -> None: ], "SELECT metric1, metric2, label1, label2 FROM table" " WHERE x < :param1 AND y > :param2", - parameters={"param1": 1, "param2": 2}, + parameter_sets=[ + {"param1": 1, "param2": 2}, + {"param1": 3, "param2": 4}, + ], ) - assert query.parameters == {"param1": 1, "param2": 2} + qe_exec1, qe_exec2 = query.executions + assert qe_exec1.name == "query[params1]" + assert qe_exec1.parameters == {"param1": 1, "param2": 2} + assert qe_exec2.name == "query[params2]" + assert qe_exec2.parameters == {"param1": 3, "param2": 4} def test_instantiate_parameters_not_matching(self) -> None: with pytest.raises(InvalidQueryParameters): @@ -116,7 +113,7 @@ def test_instantiate_parameters_not_matching(self) -> None: ], "SELECT metric1, metric2, label1, label2 FROM table" " WHERE x < :param1 AND y > :param3", - parameters={"param1": 1, "param2": 2}, + parameter_sets=[{"param1": 1, "param2": 2}], ) def test_instantiate_with_interval(self) -> None: @@ -498,8 +495,9 @@ async def test_execute_log( [QueryMetric("metric", [])], "SELECT 1.0 AS metric", ) + [query_execution] = query.executions await db.connect() - await db.execute(query) + await db.execute(query_execution) assert log.has("run query", query="query", database="db") assert log.has("action received", worker_id=ANY, action="_execute") await db.close() @@ -518,9 +516,10 @@ async def test_execute_keep_connected( [QueryMetric("metric", [])], "SELECT 1.0 AS metric", ) + [query_execution] = query.executions await db.connect() mock_conn_detach = mocker.patch.object(db._conn._conn, "detach") - await db.execute(query) + await db.execute(query_execution) assert db.connected == connected if not connected: mock_conn_detach.assert_called_once() @@ -539,21 +538,26 @@ async def test_execute_no_keep_disconnect_after_pending_queries( [QueryMetric("metric1", [])], "SELECT 1.0 AS metric1", ) + [query_execution1] = query1.executions query2 = Query( "query1", ["db"], [QueryMetric("metric2", [])], "SELECT 1.0 AS metric2", ) + [query_execution2] = query2.executions await db.connect() - await asyncio.gather(db.execute(query1), db.execute(query2)) + await asyncio.gather( + db.execute(query_execution1), db.execute(query_execution2) + ) assert not db.connected async def test_execute_not_connected(self, db: DataBase) -> None: query = Query( "query", ["db"], [QueryMetric("metric", [])], "SELECT 1 AS metric" ) - metric_results = await db.execute(query) + [query_execution] = query.executions + metric_results = await db.execute(query_execution) assert metric_results.results == [MetricResult("metric", 1, {})] # the connection is kept for reuse assert db.connected @@ -569,8 +573,9 @@ async def test_execute(self, db: DataBase) -> None: [QueryMetric("metric1", []), QueryMetric("metric2", [])], sql, ) + [query_execution] = query.executions await db.connect() - metric_results = await db.execute(query) + metric_results = await db.execute(query_execution) assert metric_results.results == [ MetricResult("metric1", 10, {}), MetricResult("metric2", 20, {}), @@ -598,8 +603,9 @@ async def test_execute_with_labels(self, db: DataBase) -> None: ], sql, ) + [query_execution] = query.executions await db.connect() - metric_results = await db.execute(query) + metric_results = await db.execute(query_execution) assert metric_results.results == [ MetricResult("metric1", 22, {"label1": "bar", "label2": "foo"}), MetricResult("metric2", 11, {"label2": "foo"}), @@ -609,9 +615,10 @@ async def test_execute_with_labels(self, db: DataBase) -> None: async def test_execute_fail(self, db: DataBase) -> None: query = Query("query", ["db"], [QueryMetric("metric", [])], "WRONG") + [query_execution] = query.executions await db.connect() with pytest.raises(DataBaseQueryError) as error: - await db.execute(query) + await db.execute(query_execution) assert "syntax error" in str(error.value) async def test_execute_query_invalid_count( @@ -623,9 +630,10 @@ async def test_execute_query_invalid_count( [QueryMetric("metric", [])], "SELECT 1 AS metric, 2 AS other", ) + [query_execution] = query.executions await db.connect() with pytest.raises(DataBaseQueryError) as error: - await db.execute(query) + await db.execute(query_execution) assert ( str(error.value) == "Wrong result count from query: expected 1, got 2" @@ -648,9 +656,10 @@ async def test_execute_query_invalid_count_with_labels( [QueryMetric("metric", ["label"])], "SELECT 1 as metric", ) + [query_execution] = query.executions await db.connect() with pytest.raises(DataBaseQueryError) as error: - await db.execute(query) + await db.execute(query_execution) assert ( str(error.value) == "Wrong result count from query: expected 2, got 1" @@ -666,9 +675,10 @@ async def test_execute_invalid_names_with_labels( [QueryMetric("metric", ["label"])], 'SELECT 1 AS foo, "bar" AS label', ) + [query_execution] = query.executions await db.connect() with pytest.raises(DataBaseQueryError) as error: - await db.execute(query) + await db.execute(query_execution) assert ( str(error.value) == "Wrong column names from query: expected (label, metric), got (foo, label)" @@ -684,12 +694,13 @@ async def test_execute_debug_exception( [QueryMetric("metric", [])], "SELECT 1 AS metric", ) + [query_execution] = query.executions await db.connect() exception = Exception("boom!") mocker.patch.object(db, "execute_sql", side_effect=exception) with pytest.raises(DataBaseQueryError) as error: - await db.execute(query) + await db.execute(query_execution) assert str(error.value) == "boom!" assert not error.value.fatal assert log.has( @@ -710,6 +721,7 @@ async def test_execute_timeout( "SELECT 1 AS metric", timeout=0.1, ) + [query_execution] = query.executions await db.connect() async def execute( @@ -721,7 +733,7 @@ async def execute( mocker.patch.object(db._conn, "execute", execute) with pytest.raises(QueryTimeoutExpired): - await db.execute(query) + await db.execute(query_execution) assert log.has( "query timeout", query="query", From c092fbe9b976e163ae6fa384f093c4e173d6c58d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:13:36 +0100 Subject: [PATCH 091/110] Bump structlog from 24.4.0 to 25.1.0 in the major group (#228) Bumps the major group with 1 update: [structlog](https://github.com/hynek/structlog). Updates `structlog` from 24.4.0 to 25.1.0 - [Release notes](https://github.com/hynek/structlog/releases) - [Changelog](https://github.com/hynek/structlog/blob/main/CHANGELOG.md) - [Commits](https://github.com/hynek/structlog/compare/24.4.0...25.1.0) --- updated-dependencies: - dependency-name: structlog dependency-type: direct:production update-type: version-update:semver-major dependency-group: major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 373b1ef..67af4e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -85,7 +85,7 @@ six==1.17.0 # via python-dateutil sqlalchemy==2.0.37 # via query-exporter (pyproject.toml) -structlog==24.4.0 +structlog==25.1.0 # via # prometheus-aioexporter # query-exporter (pyproject.toml) From 9ae1cf7e0372fa45c0c3eb5e20a476ed512fe470 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Tue, 21 Jan 2025 18:08:47 +0100 Subject: [PATCH 092/110] chore: add discussions to the doc --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 4c78f76..64218a2 100644 --- a/README.rst +++ b/README.rst @@ -262,6 +262,8 @@ Contributing The project welcomes contributions of any form. Please refer to the `contribution guide`_ for details on how to contribute. +For general purpose questions, you can use `Discussions`_ on GitHub. + .. _Prometheus: https://prometheus.io/ .. _SQLAlchemy: https://www.sqlalchemy.org/ @@ -274,6 +276,7 @@ The project welcomes contributions of any form. Please refer to the .. _`contribution guide`: docs/contributing.rst .. _`Helm chart`: https://github.com/makezbs/helm-charts/tree/main/charts/query-exporter .. _`GitHub container registry`: https://github.com/albertodonato/query-exporter/pkgs/container/query-exporter +.. _`Discussions`: https://github.com/albertodonato/query-exporter/discussions`` .. |query-exporter logo| image:: https://raw.githubusercontent.com/albertodonato/query-exporter/main/logo.svg :alt: query-exporter logo From e003f366424fd8cab4269d70032eb7e47442b5cc Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 22 Jan 2025 10:10:47 +0100 Subject: [PATCH 093/110] feat: add query interval metric (fixes #225) (#230) --- README.rst | 3 + pyproject.toml | 3 - query_exporter/main.py | 24 ++++++- query_exporter/metrics.py | 15 ++++- tests/config_test.py | 133 +++++++++++++++----------------------- tests/conftest.py | 62 +++++++++++++++++- tests/loop_test.py | 29 +-------- tests/main_test.py | 92 ++++++++++++++++++++++++++ 8 files changed, 243 insertions(+), 118 deletions(-) create mode 100644 tests/main_test.py diff --git a/README.rst b/README.rst index 64218a2..3f5e3bc 100644 --- a/README.rst +++ b/README.rst @@ -158,6 +158,9 @@ The exporter provides a few builtin metrics which can be useful to track query e ``queries{database="db",query="q",status="[success|error|timeout]"}``: a counter with number of executed queries, per database, query and status. +``query_interval{query="q"}``: + a gauge reporting the configured execution interval in seconds, if set, per query. + ``query_latency{database="db",query="q"}``: a histogram with query latencies, per database and query. diff --git a/pyproject.toml b/pyproject.toml index d10b9d5..aff491c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,9 +92,6 @@ skip_covered = true [tool.coverage.run] source = [ "query_exporter" ] -omit = [ - "query_exporter/main.py", -] [tool.mypy] ignore_missing_imports = true diff --git a/query_exporter/main.py b/query_exporter/main.py index 0873df4..95c03b4 100644 --- a/query_exporter/main.py +++ b/query_exporter/main.py @@ -2,6 +2,7 @@ from functools import partial from pathlib import Path +import typing as t from aiohttp.web import Application import click @@ -11,6 +12,7 @@ MetricConfig, PrometheusExporterScript, ) +from prometheus_client.metrics import Gauge from . import __version__ from .config import ( @@ -19,6 +21,7 @@ load_config, ) from .loop import QueryLoop +from .metrics import QUERY_INTERVAL_METRIC_NAME class QueryExporterScript(PrometheusExporterScript): @@ -60,8 +63,11 @@ def configure(self, args: Arguments) -> None: if args.check_only: raise SystemExit(0) self.create_metrics(self.config.metrics.values()) + self._set_static_metrics() - async def on_application_startup(self, application: Application) -> None: + async def on_application_startup( + self, application: Application + ) -> None: # pragma: nocover query_loop = QueryLoop(self.config, self.registry, self.logger) application["exporter"].set_metric_update_handler( partial(self._update_handler, query_loop) @@ -69,12 +75,14 @@ async def on_application_startup(self, application: Application) -> None: application["query-loop"] = query_loop await query_loop.start() - async def on_application_shutdown(self, application: Application) -> None: + async def on_application_shutdown( + self, application: Application + ) -> None: # pragma: nocover await application["query-loop"].stop() async def _update_handler( self, query_loop: QueryLoop, metrics: list[MetricConfig] - ) -> None: + ) -> None: # pragma: nocover """Run queries with no specified schedule on each request.""" await query_loop.run_aperiodic_queries() query_loop.clear_expired_series() @@ -87,5 +95,15 @@ def _load_config(self, paths: list[Path]) -> Config: self.logger.error("invalid config", error=str(error)) raise SystemExit(1) + def _set_static_metrics(self) -> None: + query_interval_metric = t.cast( + Gauge, self.registry.get_metric(QUERY_INTERVAL_METRIC_NAME) + ) + for query in self.config.queries.values(): + if query.interval: + query_interval_metric.labels(query=query.name).set( + query.interval + ) + script = QueryExporterScript() diff --git a/query_exporter/metrics.py b/query_exporter/metrics.py index 45cf620..8fe57ba 100644 --- a/query_exporter/metrics.py +++ b/query_exporter/metrics.py @@ -34,10 +34,21 @@ type="histogram", labels=("query",), ) +# metrics reporting the query interval +QUERY_INTERVAL_METRIC_NAME = "query_interval" +_QUERY_INTERVAL_METRIC_CONFIG = MetricConfig( + name=QUERY_INTERVAL_METRIC_NAME, + description="Query execution interval", + type="gauge", + labels=("query",), +) + + BUILTIN_METRICS = frozenset( ( DB_ERRORS_METRIC_NAME, QUERIES_METRIC_NAME, + QUERY_INTERVAL_METRIC_NAME, QUERY_LATENCY_METRIC_NAME, QUERY_TIMESTAMP_METRIC_NAME, ) @@ -48,7 +59,7 @@ def get_builtin_metric_configs( extra_labels: frozenset[str], ) -> dict[str, MetricConfig]: """Return configuration for builtin metrics.""" - return { + metric_configs = { metric_config.name: MetricConfig( metric_config.name, metric_config.description, @@ -63,3 +74,5 @@ def get_builtin_metric_configs( _QUERY_TIMESTAMP_METRIC_CONFIG, ) } + metric_configs[QUERY_INTERVAL_METRIC_NAME] = _QUERY_INTERVAL_METRIC_CONFIG + return metric_configs diff --git a/tests/config_test.py b/tests/config_test.py index 567bc39..2664cca 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -1,11 +1,8 @@ -from collections.abc import Callable, Iterator from pathlib import Path import typing as t -import uuid import pytest from pytest_structlog import StructuredLogCapture -import yaml from query_exporter.config import ( ConfigError, @@ -20,35 +17,7 @@ QUERIES_METRIC_NAME, ) - -@pytest.fixture -def config_full() -> Iterator[dict[str, t.Any]]: - yield { - "databases": {"db": {"dsn": "sqlite://"}}, - "metrics": {"m": {"type": "gauge", "labels": ["l1", "l2"]}}, - "queries": { - "q": { - "interval": 10, - "databases": ["db"], - "metrics": ["m"], - "sql": "SELECT 1 AS m", - } - }, - } - - -ConfigWriter = Callable[[t.Any], Path] - - -@pytest.fixture -def write_config(tmp_path: Path) -> Iterator[ConfigWriter]: - def write(data: t.Any) -> Path: - path = tmp_path / f"{uuid.uuid4()}.yaml" - path.write_text(yaml.dump(data), "utf-8") - return path - - yield write - +from .conftest import ConfigWriter CONFIG_UNKNOWN_DBS = { "databases": {}, @@ -478,12 +447,12 @@ def test_load_metrics_overlap_database_label( @pytest.mark.parametrize("builtin_metric_name", list(BUILTIN_METRICS)) def test_load_metrics_reserved_name( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, builtin_metric_name: str, ) -> None: - config_full["metrics"][builtin_metric_name] = {"type": "counter"} - config_file = write_config(config_full) + sample_config["metrics"][builtin_metric_name] = {"type": "counter"} + config_file = write_config(sample_config) with pytest.raises(ConfigError) as err: load_config([config_file]) assert ( @@ -716,11 +685,11 @@ def test_load_queries_section_invalid_schedule( def test_load_queries_section_timeout( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - config_full["queries"]["q"]["timeout"] = 2.0 - config_file = write_config(config_full) + sample_config["queries"]["q"]["timeout"] = 2.0 + config_file = write_config(sample_config) result = load_config([config_file]) query1 = result.queries["q"] assert query1.timeout == 2.0 @@ -744,13 +713,13 @@ def test_load_queries_section_timeout( ) def test_load_queries_section_invalid_timeout( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, timeout: float, error_message: str, ) -> None: - config_full["queries"]["q"]["timeout"] = timeout - config_file = write_config(config_full) + sample_config["queries"]["q"]["timeout"] = timeout + config_file = write_config(sample_config) with pytest.raises(ConfigError) as err: load_config([config_file]) assert str(err.value) == error_message @@ -804,14 +773,14 @@ def test_configuration_incorrect( def test_configuration_warning_unused( self, log: StructuredLogCapture, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - config_full["databases"]["db2"] = {"dsn": "sqlite://"} - config_full["databases"]["db3"] = {"dsn": "sqlite://"} - config_full["metrics"]["m2"] = {"type": "gauge"} - config_full["metrics"]["m3"] = {"type": "gauge"} - config_file = write_config(config_full) + sample_config["databases"]["db2"] = {"dsn": "sqlite://"} + sample_config["databases"]["db3"] = {"dsn": "sqlite://"} + sample_config["metrics"]["m2"] = {"type": "gauge"} + sample_config["metrics"]["m3"] = {"type": "gauge"} + config_file = write_config(sample_config) load_config([config_file]) assert log.has( "unused config entries", @@ -850,35 +819,35 @@ def test_load_queries_missing_interval_default_to_none( ) def test_load_queries_interval( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, interval: str | int | None, value: int | None, ) -> None: - config_full["queries"]["q"]["interval"] = interval - config_file = write_config(config_full) + sample_config["queries"]["q"]["interval"] = interval + config_file = write_config(sample_config) config = load_config([config_file]) assert config.queries["q"].interval == value def test_load_queries_interval_not_specified( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - del config_full["queries"]["q"]["interval"] - config_file = write_config(config_full) + del sample_config["queries"]["q"]["interval"] + config_file = write_config(sample_config) config = load_config([config_file]) assert config.queries["q"].interval is None @pytest.mark.parametrize("interval", ["1x", "wrong", "1.5m"]) def test_load_queries_invalid_interval_string( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, interval: str, ) -> None: - config_full["queries"]["q"]["interval"] = interval - config_file = write_config(config_full) + sample_config["queries"]["q"]["interval"] = interval + config_file = write_config(sample_config) with pytest.raises(ConfigError) as err: load_config([config_file]) assert str(err.value) == ( @@ -889,12 +858,12 @@ def test_load_queries_invalid_interval_string( @pytest.mark.parametrize("interval", [0, -20]) def test_load_queries_invalid_interval_number( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, interval: int, ) -> None: - config_full["queries"]["q"]["interval"] = interval - config_file = write_config(config_full) + sample_config["queries"]["q"]["interval"] = interval + config_file = write_config(sample_config) with pytest.raises(ConfigError) as err: load_config([config_file]) assert ( @@ -904,11 +873,11 @@ def test_load_queries_invalid_interval_number( def test_load_queries_no_metrics( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - config_full["queries"]["q"]["metrics"] = [] - config_file = write_config(config_full) + sample_config["queries"]["q"]["metrics"] = [] + config_file = write_config(sample_config) with pytest.raises(ConfigError) as err: load_config([config_file]) assert ( @@ -918,11 +887,11 @@ def test_load_queries_no_metrics( def test_load_queries_no_databases( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - config_full["queries"]["q"]["databases"] = [] - config_file = write_config(config_full) + sample_config["queries"]["q"]["databases"] = [] + config_file = write_config(sample_config) with pytest.raises(ConfigError) as err: load_config([config_file]) assert ( @@ -944,25 +913,25 @@ def test_load_queries_no_databases( ) def test_load_metrics_expiration( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, expiration: str | int | None, value: int | None, ) -> None: - config_full["metrics"]["m"]["expiration"] = expiration - config_file = write_config(config_full) + sample_config["metrics"]["m"]["expiration"] = expiration + config_file = write_config(sample_config) config = load_config([config_file]) assert config.metrics["m"].config["expiration"] == value def test_load_multiple_files( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - file_full = write_config(config_full) - file1 = write_config({"databases": config_full["databases"]}) - file2 = write_config({"metrics": config_full["metrics"]}) - file3 = write_config({"queries": config_full["queries"]}) + file_full = write_config(sample_config) + file1 = write_config({"databases": sample_config["databases"]}) + file2 = write_config({"metrics": sample_config["metrics"]}) + file3 = write_config({"queries": sample_config["queries"]}) assert load_config([file1, file2, file3]) == load_config([file_full]) def test_load_multiple_files_combine( @@ -1002,11 +971,11 @@ def test_load_multiple_files_combine( def test_load_multiple_files_duplicated_database( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - file1 = write_config(config_full) - file2 = write_config({"databases": config_full["databases"]}) + file1 = write_config(sample_config) + file2 = write_config({"databases": sample_config["databases"]}) with pytest.raises(ConfigError) as err: load_config([file1, file2]) assert ( @@ -1016,11 +985,11 @@ def test_load_multiple_files_duplicated_database( def test_load_multiple_files_duplicated_metric( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - file1 = write_config(config_full) - file2 = write_config({"metrics": config_full["metrics"]}) + file1 = write_config(sample_config) + file2 = write_config({"metrics": sample_config["metrics"]}) with pytest.raises(ConfigError) as err: load_config([file1, file2]) assert ( @@ -1029,11 +998,11 @@ def test_load_multiple_files_duplicated_metric( def test_load_multiple_files_duplicated_query( self, - config_full: dict[str, t.Any], + sample_config: dict[str, t.Any], write_config: ConfigWriter, ) -> None: - file1 = write_config(config_full) - file2 = write_config({"queries": config_full["queries"]}) + file1 = write_config(sample_config) + file2 = write_config({"queries": sample_config["queries"]}) with pytest.raises(ConfigError) as err: load_config([file1, file2]) assert ( diff --git a/tests/conftest.py b/tests/conftest.py index 46cbd80..d90229b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,14 +1,20 @@ import asyncio +from collections import defaultdict from collections.abc import AsyncIterator, Iterator +from pathlib import Path +import typing as t +import uuid +from prometheus_client.metrics import MetricWrapperBase import pytest from pytest_mock import MockerFixture from pytest_structlog import StructuredLogCapture from toolrack.testing.fixtures import advance_time +import yaml from query_exporter.db import DataBase, MetricResults, QueryExecution -__all__ = ["QueryTracker", "advance_time", "query_tracker"] +__all__ = ["MetricValues", "QueryTracker", "advance_time", "metric_values"] @pytest.fixture(autouse=True) @@ -65,3 +71,57 @@ async def execute( mocker.patch.object(DataBase, "execute", execute) yield tracker + + +@pytest.fixture +def sample_config() -> Iterator[dict[str, t.Any]]: + yield { + "databases": {"db": {"dsn": "sqlite://"}}, + "metrics": {"m": {"type": "gauge", "labels": ["l1", "l2"]}}, + "queries": { + "q": { + "interval": 10, + "databases": ["db"], + "metrics": ["m"], + "sql": "SELECT 1 AS m", + } + }, + } + + +ConfigWriter = t.Callable[[t.Any], Path] + + +@pytest.fixture +def write_config(tmp_path: Path) -> Iterator[ConfigWriter]: + def write(data: t.Any) -> Path: + path = tmp_path / f"{uuid.uuid4()}.yaml" + path.write_text(yaml.dump(data), "utf-8") + return path + + yield write + + +MetricValues = list[int | float] | dict[tuple[str, ...], int | float] + + +def metric_values( + metric: MetricWrapperBase, by_labels: tuple[str, ...] = () +) -> MetricValues: + """Return values for the metric.""" + if metric._type == "gauge": + suffix = "" + elif metric._type == "counter": + suffix = "_total" + + values_by_label: dict[tuple[str, ...], int | float] = {} + values_by_suffix: dict[str, list[int | float]] = defaultdict(list) + for sample_suffix, labels, value, *_ in metric._samples(): + if sample_suffix == suffix: + if by_labels: + label_values = tuple(labels[label] for label in by_labels) + values_by_label[label_values] = value + else: + values_by_suffix[sample_suffix].append(value) + + return values_by_label if by_labels else values_by_suffix[suffix] diff --git a/tests/loop_test.py b/tests/loop_test.py index 394e782..ce41a85 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -1,5 +1,4 @@ import asyncio -from collections import defaultdict from collections.abc import AsyncIterator, Callable, Iterator from decimal import Decimal from pathlib import Path @@ -7,7 +6,6 @@ from unittest.mock import ANY from prometheus_aioexporter import MetricsRegistry -from prometheus_client.metrics import MetricWrapperBase import pytest from pytest_mock import MockerFixture from pytest_structlog import StructuredLogCapture @@ -18,7 +16,7 @@ from query_exporter.db import DataBase, DataBaseConfig from query_exporter.loop import MetricsLastSeen, QueryLoop -from .conftest import QueryTracker +from .conftest import QueryTracker, metric_values AdvanceTime = Callable[[float], t.Awaitable[None]] @@ -76,31 +74,6 @@ async def query_loop( yield make_query_loop() -MetricValues = list[int | float] | dict[tuple[str, ...], int | float] - - -def metric_values( - metric: MetricWrapperBase, by_labels: tuple[str, ...] = () -) -> MetricValues: - """Return values for the metric.""" - if metric._type == "gauge": - suffix = "" - elif metric._type == "counter": - suffix = "_total" - - values_by_label: dict[tuple[str, ...], int | float] = {} - values_by_suffix: dict[str, list[int | float]] = defaultdict(list) - for sample_suffix, labels, value, *_ in metric._samples(): - if sample_suffix == suffix: - if by_labels: - label_values = tuple(labels[label] for label in by_labels) - values_by_label[label_values] = value - else: - values_by_suffix[sample_suffix].append(value) - - return values_by_label if by_labels else values_by_suffix[suffix] - - async def run_queries(db_file: Path, *queries: str) -> None: config = DataBaseConfig(name="db", dsn=f"sqlite:///{db_file}") async with DataBase(config) as db: diff --git a/tests/main_test.py b/tests/main_test.py new file mode 100644 index 0000000..a1e7a7a --- /dev/null +++ b/tests/main_test.py @@ -0,0 +1,92 @@ +from collections.abc import Iterator +from copy import deepcopy +import typing as t +from unittest import mock + +from click.testing import CliRunner, Result +import pytest +from pytest_mock import MockerFixture + +from query_exporter.main import QueryExporterScript + +from .conftest import ConfigWriter, metric_values + + +@pytest.fixture +def mock_run_app(mocker: MockerFixture) -> Iterator[mock.MagicMock]: + yield mocker.patch("prometheus_aioexporter._web.run_app") + + +@pytest.fixture +def script() -> Iterator[QueryExporterScript]: + yield QueryExporterScript() + + +@pytest.fixture +def invoke_cli( + mock_run_app: mock.MagicMock, + script: QueryExporterScript, +) -> Iterator[t.Callable[..., Result]]: + def invoke(*args: str) -> Result: + return CliRunner().invoke(script.command, args) + + yield invoke + + +class TestQureyExporterScript: + def test_run( + self, + mock_run_app: mock.MagicMock, + sample_config: dict[str, t.Any], + write_config: ConfigWriter, + invoke_cli: t.Callable[..., Result], + ) -> None: + config_file = write_config(sample_config) + invoke_cli("--config", str(config_file)) + mock_run_app.assert_called_once() + + def test_run_check_only( + self, + mock_run_app: mock.MagicMock, + sample_config: dict[str, t.Any], + write_config: ConfigWriter, + invoke_cli: t.Callable[..., Result], + ) -> None: + config_file = write_config(sample_config) + result = invoke_cli("--config", str(config_file), "--check-only") + assert result.exit_code == 0 + mock_run_app.assert_not_called() + + def test_run_check_only_wrong_config( + self, + mock_run_app: mock.MagicMock, + sample_config: dict[str, t.Any], + write_config: ConfigWriter, + invoke_cli: t.Callable[..., Result], + ) -> None: + sample_config["extra"] = "stuff" + config_file = write_config(sample_config) + result = invoke_cli("--config", str(config_file), "--check-only") + assert result.exit_code == 1 + mock_run_app.assert_not_called() + + def test_static_metrics_query_interval( + self, + mock_run_app: mock.MagicMock, + script: QueryExporterScript, + sample_config: dict[str, t.Any], + write_config: ConfigWriter, + invoke_cli: t.Callable[..., Result], + ) -> None: + sample_config["queries"]["q2"] = deepcopy( + sample_config["queries"]["q"] + ) + sample_config["queries"]["q2"]["interval"] = 20 + config_file = write_config(sample_config) + result = invoke_cli("--config", str(config_file)) + assert result.exit_code == 0 + metric = script.registry.get_metric("query_interval") + assert metric_values(metric, by_labels=("query",)) == { + ("q",): 10.0, + ("q2",): 20.0, + } From 30421c5c6d06f2dfbd71f3471d62e45285ab712d Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 22 Jan 2025 12:09:39 +0100 Subject: [PATCH 094/110] chore: fix url --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3f5e3bc..8015f7e 100644 --- a/README.rst +++ b/README.rst @@ -279,7 +279,7 @@ For general purpose questions, you can use `Discussions`_ on GitHub. .. _`contribution guide`: docs/contributing.rst .. _`Helm chart`: https://github.com/makezbs/helm-charts/tree/main/charts/query-exporter .. _`GitHub container registry`: https://github.com/albertodonato/query-exporter/pkgs/container/query-exporter -.. _`Discussions`: https://github.com/albertodonato/query-exporter/discussions`` +.. _`Discussions`: https://github.com/albertodonato/query-exporter/discussions .. |query-exporter logo| image:: https://raw.githubusercontent.com/albertodonato/query-exporter/main/logo.svg :alt: query-exporter logo From d1647cb4178dda1261734e98506dd5398ad11da6 Mon Sep 17 00:00:00 2001 From: Alberto Donato Date: Wed, 22 Jan 2025 14:47:50 +0100 Subject: [PATCH 095/110] feat: run queries in explicit transactions, deprecate autocommit flag (#232) --- docs/configuration.rst | 3 ++ docs/databases.rst | 10 ------- query_exporter/config.py | 7 +++++ query_exporter/db.py | 47 ++++++++++++++++++------------ query_exporter/schemas/config.yaml | 5 ++-- tests/db_test.py | 10 +++---- 6 files changed, 47 insertions(+), 35 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index dd26d18..d1423cd 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -85,6 +85,9 @@ Each database definitions can have the following keys: defaults to ``true``. This should only be changed to ``false`` if specific queries require it. + **Note**: this option is deprecated and be removed in the next major + release. Explicit transactions are now always used to run each query. + ``labels``: an optional mapping of label names and values to tag metrics collected from each database. When labels are used, all databases must define the same set of labels. diff --git a/docs/databases.rst b/docs/databases.rst index 131ace0..ae45845 100644 --- a/docs/databases.rst +++ b/docs/databases.rst @@ -25,16 +25,6 @@ This can be done via the ``MARS_Connection`` parameter on the database DSN:: mssql://:@:/?MARS_Connection=yes -Connection closed error during query execution ----------------------------------------------- - -In some cases an error about ``"The cursor's connection has been closed"`` is -returned when fetching query results. - -If this happens, setting ``autocommit: false`` in database configuration should -solve it. - - Oracle [``oracle://``] ====================== diff --git a/query_exporter/config.py b/query_exporter/config.py index cf5e620..1ce93a7 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -131,6 +131,13 @@ def _get_databases( keep_connected=config.get("keep-connected", True), autocommit=config.get("autocommit", True), ) + if "autocommit" in config: + logger = structlog.get_logger() + logger.warn( + f"deprecated 'autocommit' option for database '{name}'", + database=name, + ) + except Exception as e: raise ConfigError(str(e)) diff --git a/query_exporter/db.py b/query_exporter/db.py index 3d764ff..3f4aa19 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -39,6 +39,7 @@ ArgumentError, NoSuchModuleError, ) +from sqlalchemy.pool import NullPool from sqlalchemy.sql.elements import TextClause import structlog @@ -134,10 +135,10 @@ def __post_init__(self) -> None: create_db_engine(self.dsn) -def create_db_engine(dsn: str, **kwargs: t.Any) -> Engine: - """Create the database engine, validating the DSN""" +def create_db_engine(dsn: str) -> Engine: + """Create the database engine, validating the DSN.""" try: - return create_engine(dsn, **kwargs) + return create_engine(dsn, poolclass=NullPool) except ImportError as error: raise DataBaseError(f'module "{error.name}" not found') except (ArgumentError, ValueError, NoSuchModuleError): @@ -355,7 +356,7 @@ async def close(self) -> None: async def execute( self, - sql: TextClause, + statement: TextClause, parameters: dict[str, t.Any] | None = None, ) -> QueryResults: """Execute a query, returning results.""" @@ -363,9 +364,16 @@ async def execute( parameters = {} return t.cast( QueryResults, - await self._call_in_thread(self._execute, sql, parameters), + await self._call_in_thread(self._execute, statement, parameters), ) + async def execute_many( + self, + statements: list[TextClause], + ) -> None: + """Execute multiple statements.""" + await self._call_in_thread(self._execute_many, statements) + def _create_worker(self) -> None: assert not self._worker self._worker = Thread( @@ -382,11 +390,18 @@ def _connect(self) -> None: self._conn = self.engine.connect() def _execute( - self, sql: TextClause, parameters: dict[str, t.Any] + self, statement: TextClause, parameters: dict[str, t.Any] ) -> QueryResults: assert self._conn - result = self._conn.execute(sql, parameters) - return QueryResults.from_result(result) + with self._conn.begin(): + result = self._conn.execute(statement, parameters) + return QueryResults.from_result(result) + + def _execute_many(self, statements: list[TextClause]) -> None: + assert self._conn + with self._conn.begin(): + for statement in statements: + self._conn.execute(statement) def _close(self) -> None: assert self._conn @@ -436,13 +451,7 @@ def __init__( logger = structlog.get_logger() self.logger = logger.bind(database=self.config.name) self._connect_lock = asyncio.Lock() - execution_options = {} - if self.config.autocommit: - execution_options["isolation_level"] = "AUTOCOMMIT" - engine = create_db_engine( - self.config.dsn, - execution_options=execution_options, - ) + engine = create_db_engine(self.config.dsn) self._conn = DataBaseConnection(self.config.name, engine, self.logger) self._setup_query_latency_tracking(engine) @@ -475,13 +484,15 @@ async def connect(self) -> None: raise self._db_error(error, exc_class=DataBaseConnectError) self.logger.debug("connected") - for sql in self.config.connect_sql: + if self.config.connect_sql: try: - await self.execute_sql(sql) + await self._conn.execute_many( + [text(sql) for sql in self.config.connect_sql] + ) except Exception as error: await self._close() raise self._db_error( - f'failed executing query "{sql}": {error}', + f"failed executing connect SQL: {error}", exc_class=DataBaseQueryError, ) diff --git a/query_exporter/schemas/config.yaml b/query_exporter/schemas/config.yaml index e03b82a..76a6d26 100644 --- a/query_exporter/schemas/config.yaml +++ b/query_exporter/schemas/config.yaml @@ -114,9 +114,10 @@ definitions: autocommit: title: Whether to enable autocommit for queries description: > - When set to false, don't autocommit after each query. + NOTE: setting autocommit on or off no longer has effect since each + query is run in a separate transaction. This option is deprecated + and will be removed in the next major release. type: boolean - default: true keep-connected: title: Whether to keep the connection open for the database between queries description: > diff --git a/tests/db_test.py b/tests/db_test.py index 1c3d570..af248fc 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -452,12 +452,12 @@ async def test_connect_sql(self, mocker: MockerFixture) -> None: ) db = DataBase(config) - queries = [] + queries: list[str] = [] - async def execute_sql(sql: str) -> None: - queries.append(sql) + async def execute_many(statements: list[TextClause]) -> None: + queries.extend(statement.text for statement in statements) - mocker.patch.object(db, "execute_sql", execute_sql) + mocker.patch.object(db._conn, "execute_many", execute_many) await db.connect() assert queries == ["SELECT 1", "SELECT 2"] await db.close() @@ -472,7 +472,7 @@ async def test_connect_sql_fail(self, log: StructuredLogCapture) -> None: with pytest.raises(DataBaseQueryError) as error: await db.connect() assert not db.connected - assert 'failed executing query "WRONG"' in str(error.value) + assert "failed executing connect SQL" in str(error.value) assert log.has("disconnected", database="db") async def test_close( From cc67ec07f478cb98dcdaf56305d92afd845f5868 Mon Sep 17 00:00:00 2001 From: Leo Yorke Date: Mon, 14 Aug 2023 14:21:20 -0500 Subject: [PATCH 096/110] Add the ability to connect to snowflake --- Dockerfile | 4 +- pyproject.toml | 32 ++++------- query_exporter/config.py | 1 + query_exporter/db.py | 89 +++++++++++++++++++++++++++++- query_exporter/loop.py | 13 ++++- query_exporter/main.py | 4 +- query_exporter/schemas/config.yaml | 4 ++ requirements.txt | 59 +++++++++++++++++--- 8 files changed, 169 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 413acbe..dbab8ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,9 +23,7 @@ RUN pip install \ clickhouse-sqlalchemy \ "ibm-db-sa; platform_machine == 'x86_64' or platform_machine == 'ppc64le'" \ mysqlclient \ - psycopg2 \ - pymssql \ - pyodbc + psycopg2 RUN curl \ https://download.oracle.com/otn_software/linux/instantclient/instantclient-basiclite-linux$(arch | sed -e 's/x86_64/x64/g; s/aarch64/-arm64/g').zip \ diff --git a/pyproject.toml b/pyproject.toml index aff491c..8cfd097 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,26 +1,15 @@ [build-system] build-backend = "setuptools.build_meta" -requires = [ - "setuptools", -] +requires = ["setuptools"] [project] name = "query-exporter" description = "Export Prometheus metrics generated from SQL queries" readme = "README.rst" -keywords = [ - "exporter", - "metric", - "prometheus", - "sql", -] +keywords = ["exporter", "metric", "prometheus", "sql"] license = { file = "LICENSE.txt" } -maintainers = [ - { name = "Alberto Donato", email = "alberto.donato@gmail.com" }, -] -authors = [ - { name = "Alberto Donato", email = "alberto.donato@gmail.com" }, -] +maintainers = [{ name = "Alberto Donato", email = "alberto.donato@gmail.com" }] +authors = [{ name = "Alberto Donato", email = "alberto.donato@gmail.com" }] requires-python = ">=3.11" classifiers = [ "Development Status :: 5 - Production/Stable", @@ -36,9 +25,7 @@ classifiers = [ "Topic :: System :: Monitoring", "Topic :: Utilities", ] -dynamic = [ - "version", -] +dynamic = ["version"] dependencies = [ "aiohttp", "croniter", @@ -50,6 +37,7 @@ dependencies = [ "sqlalchemy>=2", "structlog", "toolrack>=4", + "snowflake-connector-python", ] optional-dependencies.testing = [ "pytest", @@ -69,15 +57,15 @@ scripts.query-exporter = "query_exporter.main:script" version = { attr = "query_exporter.__version__" } [tool.setuptools.packages.find] -include = [ "query_exporter*" ] +include = ["query_exporter*"] [tool.setuptools.package-data] -query_exporter = [ "py.typed", "schemas/*" ] +query_exporter = ["py.typed", "schemas/*"] [tool.ruff] line-length = 79 -lint.select = [ "F", "I", "RUF", "UP" ] +lint.select = ["F", "I", "RUF", "UP"] lint.isort.combine-as-imports = true lint.isort.force-sort-within-sections = true @@ -91,7 +79,7 @@ show_missing = true skip_covered = true [tool.coverage.run] -source = [ "query_exporter" ] +source = ["query_exporter"] [tool.mypy] ignore_missing_imports = true diff --git a/query_exporter/config.py b/query_exporter/config.py index 1ce93a7..e6b7879 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -126,6 +126,7 @@ def _get_databases( databases[name] = DataBaseConfig( name, _resolve_dsn(config["dsn"], env), + conn_type=config.get("conn_type", "generic"), connect_sql=config.get("connect-sql", []), labels=labels, keep_connected=config.get("keep-connected", True), diff --git a/query_exporter/db.py b/query_exporter/db.py index 3f4aa19..5b4575d 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -6,6 +6,10 @@ Iterable, Sequence, ) +from itertools import chain +import logging +from time import perf_counter +import typing as t from dataclasses import ( InitVar, dataclass, @@ -42,6 +46,8 @@ from sqlalchemy.pool import NullPool from sqlalchemy.sql.elements import TextClause import structlog +import snowflake.connector +from snowflake.connector.connection import SnowflakeConnection # Timeout for a query QueryTimeout = int | float @@ -119,12 +125,17 @@ def __init__(self, query_name: str, message: str) -> None: FATAL_ERRORS = (InvalidResultCount, InvalidResultColumnNames) +class DatabaseConfigError(Exception): + """Database Configuration is invalid.""" + + @dataclass(frozen=True) class DataBaseConfig: """Configuration for a database.""" name: str dsn: str + conn_type: str = "generic" connect_sql: list[str] = field(default_factory=list) labels: dict[str, str] = field(default_factory=dict) keep_connected: bool = True @@ -132,7 +143,13 @@ class DataBaseConfig: def __post_init__(self) -> None: # raise DatabaseError error if the DSN in invalid - create_db_engine(self.dsn) + try: + if self.conn_type == "snowflake": + create_snowflake_connection(self.dsn) + else: + create_db_engine(self.dsn) + except DataBaseError as e: + raise DatabaseConfigError(str(e)) def create_db_engine(dsn: str) -> Engine: @@ -145,6 +162,16 @@ def create_db_engine(dsn: str) -> Engine: raise DataBaseError(f'Invalid database DSN: "{dsn}"') +def create_snowflake_connection(dsn: str) -> SnowflakeConnection: + connection_config = re.search("(snowflake:\/\/)(.+)(:)(.+)(@)(.+)", dsn) + + return snowflake.connector.connect( + user=connection_config.group(2), + password=connection_config.group(4), + account=connection_config.group(6), + ) + + class QueryMetric(t.NamedTuple): """Metric details for a Query.""" @@ -171,6 +198,12 @@ def from_result(cls, result: CursorResult[t.Any]) -> t.Self: latency = result.connection.info.get("query_latency", None) return cls(keys, rows, timestamp=timestamp, latency=latency) + @classmethod + def from_snowflake(cls, keys, results): + return cls( + [key.name.lower() for key in keys], [result for result in results] + ) + class MetricResult(t.NamedTuple): """A result for a metric from a query.""" @@ -435,6 +468,60 @@ async def _call_in_thread( return await call.result() +class SnowflakeDataBase: + _pending_queries: int = 0 + + def __init__( + self, + config, + logger: logging.Logger = logging.getLogger(), + ): + self.config = config + self.logger = logger + self._engine: SnowflakeConnection = create_snowflake_connection( + self.config.dsn + ) + + async def __aenter__(self): + if not self._engine: + self._engine: SnowflakeConnection = create_snowflake_connection( + self.config.dsn + ) + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + await self.close() + + async def close(self): + await self._engine.close() + + async def execute(self, query: Query) -> MetricResults: + """Execute a query.""" + self.logger.debug( + f'running query "{query.name}" on database "{self.config.name}"' + ) + self._pending_queries += 1 + try: + cur = self._engine.cursor() + cur.execute_async(query.sql) + cur.get_results_from_sfqid(cur.sfqid) + results = cur.fetchall() + return query.results( + QueryResults.from_snowflake(cur.description, results) + ) + except Exception as error: + message = str(error).strip() + self.logger.error( + f'error from database "{self.config.name}": {message}' + ) + raise DataBaseQueryError(message) + finally: + assert self._pending_queries >= 0, "pending queries is negative" + self._pending_queries -= 1 + if not self.config.keep_connected and not self._pending_queries: + self.close() + + class DataBase: """A database to perform Queries.""" diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 08f584c..eb81ee0 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -34,6 +34,7 @@ Query, QueryExecution, QueryTimeoutExpired, + SnowflakeDataBase, ) from .metrics import ( DB_ERRORS_METRIC_NAME, @@ -126,8 +127,16 @@ def __init__( for name, metric in self._config.metrics.items() } ) - self._databases: dict[str, DataBase] = { - db_config.name: DataBase(db_config, logger=self._logger) + + database_classes = { + "snowflake": SnowflakeDataBase, + "generic": DataBase, + } + + self._databases: dict[str, Union[DataBase, SnowflakeDataBase]] = { + db_config.name: database_classes[db_config.conn_type]( + db_config, logger=self._logger + ) for db_config in self._config.databases.values() } diff --git a/query_exporter/main.py b/query_exporter/main.py index 95c03b4..1745f9f 100644 --- a/query_exporter/main.py +++ b/query_exporter/main.py @@ -14,8 +14,8 @@ ) from prometheus_client.metrics import Gauge -from . import __version__ -from .config import ( +from query_exporter import __version__ +from query_exporter.config import ( Config, ConfigError, load_config, diff --git a/query_exporter/schemas/config.yaml b/query_exporter/schemas/config.yaml index 76a6d26..90e873c 100644 --- a/query_exporter/schemas/config.yaml +++ b/query_exporter/schemas/config.yaml @@ -118,6 +118,10 @@ definitions: query is run in a separate transaction. This option is deprecated and will be removed in the next major release. type: boolean + default: true + conn_type: + title: What type of connection this is + type: string keep-connected: title: Whether to keep the connection open for the database between queries description: > diff --git a/requirements.txt b/requirements.txt index 67af4e6..3e48c4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # # pip-compile --output-file=requirements.txt pyproject.toml @@ -12,23 +12,44 @@ aiohttp==3.11.11 # query-exporter (pyproject.toml) aiosignal==1.3.2 # via aiohttp +asn1crypto==1.5.1 + # via snowflake-connector-python attrs==24.3.0 # via # aiohttp # jsonschema # referencing +certifi==2024.12.14 + # via + # requests + # snowflake-connector-python +cffi==1.17.1 + # via + # cryptography + # snowflake-connector-python +charset-normalizer==3.4.1 + # via + # requests + # snowflake-connector-python click==8.1.8 # via prometheus-aioexporter croniter==6.0.0 # via query-exporter (pyproject.toml) +cryptography==44.0.0 + # via + # pyopenssl + # snowflake-connector-python +filelock==3.17.0 + # via snowflake-connector-python frozenlist==1.5.0 # via # aiohttp # aiosignal -greenlet==3.1.1 - # via sqlalchemy idna==3.10 - # via yarl + # via + # requests + # snowflake-connector-python + # yarl iniconfig==2.0.0 # via pytest jsonschema==4.23.0 @@ -44,7 +65,11 @@ multidict==6.1.0 # aiohttp # yarl packaging==24.2 - # via pytest + # via + # pytest + # snowflake-connector-python +platformdirs==4.3.6 + # via snowflake-connector-python pluggy==1.5.0 # via pytest prometheus-aioexporter==3.0.1 @@ -57,8 +82,14 @@ propcache==0.2.1 # via # aiohttp # yarl +pycparser==2.22 + # via cffi pygments==2.19.1 # via rich +pyjwt==2.10.1 + # via snowflake-connector-python +pyopenssl==24.3.0 + # via snowflake-connector-python pytest==8.3.4 # via toolrack python-dateutil==2.9.0.post0 @@ -68,13 +99,17 @@ python-dateutil==2.9.0.post0 python-dotenv==1.0.1 # via prometheus-aioexporter pytz==2024.2 - # via croniter + # via + # croniter + # snowflake-connector-python pyyaml==6.0.2 # via query-exporter (pyproject.toml) referencing==0.36.1 # via # jsonschema # jsonschema-specifications +requests==2.32.3 + # via snowflake-connector-python rich==13.9.4 # via prometheus-aioexporter rpds-py==0.22.3 @@ -83,15 +118,25 @@ rpds-py==0.22.3 # referencing six==1.17.0 # via python-dateutil +snowflake-connector-python==3.12.4 + # via query-exporter (pyproject.toml) +sortedcontainers==2.4.0 + # via snowflake-connector-python sqlalchemy==2.0.37 # via query-exporter (pyproject.toml) structlog==25.1.0 # via # prometheus-aioexporter # query-exporter (pyproject.toml) +tomlkit==0.13.2 + # via snowflake-connector-python toolrack==4.0.1 # via query-exporter (pyproject.toml) typing-extensions==4.12.2 - # via sqlalchemy + # via + # snowflake-connector-python + # sqlalchemy +urllib3==2.3.0 + # via requests yarl==1.18.3 # via aiohttp From c0a2684cd2fb6993e439e73a41532ed0b036d444 Mon Sep 17 00:00:00 2001 From: Leo Yorke Date: Tue, 15 Aug 2023 11:13:28 -0500 Subject: [PATCH 097/110] Infrastructure to support the query exporter. --- Dockerfile | 2 + infrastructure/aws/buildspec.yml | 39 ++++++++++++++++++++ infrastructure/aws/checkspec.yml | 27 ++++++++++++++ infrastructure/bash/docker_login.sh | 13 +++++++ infrastructure/bash/push_docker_image_ecr.sh | 30 +++++++++++++++ infrastructure/bash/run_docker_tests.sh | 10 +++++ infrastructure/bash/setup_ssh.sh | 8 ++++ infrastructure/project_config.json | 8 ++++ infrastructure/ssh/config | 2 + infrastructure/ssh/known_hosts | 1 + local-stack.yaml | 13 +++++++ testing-stack.yaml | 17 +++++++++ 12 files changed, 170 insertions(+) create mode 100644 infrastructure/aws/buildspec.yml create mode 100644 infrastructure/aws/checkspec.yml create mode 100644 infrastructure/bash/docker_login.sh create mode 100755 infrastructure/bash/push_docker_image_ecr.sh create mode 100644 infrastructure/bash/run_docker_tests.sh create mode 100755 infrastructure/bash/setup_ssh.sh create mode 100644 infrastructure/project_config.json create mode 100644 infrastructure/ssh/config create mode 100644 infrastructure/ssh/known_hosts create mode 100644 local-stack.yaml create mode 100644 testing-stack.yaml diff --git a/Dockerfile b/Dockerfile index dbab8ad..60776ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,8 @@ RUN apt-get install -y --no-install-recommends \ unzip COPY . /srcdir +WORKDIR /srcdir + RUN python3 -m venv /virtualenv ENV PATH="/virtualenv/bin:$PATH" RUN pip install \ diff --git a/infrastructure/aws/buildspec.yml b/infrastructure/aws/buildspec.yml new file mode 100644 index 0000000..6f36832 --- /dev/null +++ b/infrastructure/aws/buildspec.yml @@ -0,0 +1,39 @@ +version: 0.2 + +env: + parameter-store: + build_ssh_key: "habitat_energy_machine_account" + +phases: + install: + runtime-versions: + python: 3.10 + pre_build: + commands: + - # make environmental variables + - if [ -z "$ENV" ]; then export ENV="development"; fi + - if [ -z "$EVENT" ]; then export EVENT="$CODEBUILD_WEBHOOK_EVENT"; fi + - + - # setup ssh + - echo "$build_ssh_key" | tr -d '\r' > infrastructure/ssh/id_rsa + - bash infrastructure/bash/setup_ssh.sh + - + - # run all tests with docker file + - bash infrastructure/bash/docker_login.sh + - bash infrastructure/bash/run_docker_tests.sh + build: + commands: + # build images + - docker-compose -f local-stack.yaml -p build build --no-cache + # push to ecr + - export ECR_REPOSITORY=$(cat infrastructure/project_config.json | jq -r .ecr.repository) + - bash infrastructure/bash/push_docker_image_ecr.sh $(cat infrastructure/project_config.json | jq -r .aws.account).dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${ECR_REPOSITORY} + post_build: + commands: + - printf '[{"name":"dev-usa-query-exporter", "imageUri":"417844920390.dkr.ecr.us-east-1.amazonaws.com/usa-query-exporter:main"}]' > imagedefinitions.json +artifacts: + files: imagedefinitions.json +cache: + paths: + - '/root/.cache/pip/**/*' + - 'build/**/*' diff --git a/infrastructure/aws/checkspec.yml b/infrastructure/aws/checkspec.yml new file mode 100644 index 0000000..140c5ee --- /dev/null +++ b/infrastructure/aws/checkspec.yml @@ -0,0 +1,27 @@ +version: 0.2 + +env: + parameter-store: + build_ssh_key: "habitat_energy_machine_account" + +phases: + install: + runtime-versions: + python: 3.10 + build: + commands: + - # make environmental variables + - if [ -z "$ENV" ]; then export ENV="development"; fi + - if [ -z "$EVENT" ]; then export EVENT="$CODEBUILD_WEBHOOK_EVENT"; fi + - + - # setup ssh + - echo "$build_ssh_key" | tr -d '\r' > infrastructure/ssh/id_rsa + - bash infrastructure/bash/setup_ssh.sh + - + - # run all tests with docker file + - . infrastructure/bash/docker_login.sh + - . infrastructure/bash/run_docker_tests.sh +cache: + paths: + - '/root/.cache/pip/**/*' + - 'build/**/*' diff --git a/infrastructure/bash/docker_login.sh b/infrastructure/bash/docker_login.sh new file mode 100644 index 0000000..d6f2881 --- /dev/null +++ b/infrastructure/bash/docker_login.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex + +# ECR login +aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin "$(cat infrastructure/project_config.json | jq -r .aws.account)".dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com + +# get docker login details +export DOCKERHUB_USERNAME=`aws secretsmanager get-secret-value --secret-id docker_hub_ci_login --query 'SecretString' --output text --region ${AWS_DEFAULT_REGION} | jq -r ".username"` +export DOCKERHUB_PASSWORD=`aws secretsmanager get-secret-value --secret-id docker_hub_ci_login --query 'SecretString' --output text --region ${AWS_DEFAULT_REGION} | jq -r ".password"` +# login into docker +echo Logging in to Docker Hub... +echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin \ No newline at end of file diff --git a/infrastructure/bash/push_docker_image_ecr.sh b/infrastructure/bash/push_docker_image_ecr.sh new file mode 100755 index 0000000..caf057b --- /dev/null +++ b/infrastructure/bash/push_docker_image_ecr.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e +set -x + +# make docker container string +docker_repo="$1" + +# get version number +refs=$(git show-ref --head | python3 -c 'import sys +import os +for line in sys.stdin: + commit, ref = line.split(" ") + ref = ref.strip() + if ref == "HEAD": + head = commit + elif commit == head and ref.replace("refs/heads/", "") in ("main", "development" ): + print(ref.replace("refs/heads/", "")) + +if (tag:=os.popen("git tag --points-at HEAD").read().strip()): + print(tag.replace("refs/tags/v", "")) +') + +echo "Pushing refs" +for ref in $refs ; do + echo "Pushing $ref" + docker tag build-app:latest $docker_repo:$ref + docker push $docker_repo:$ref +done +echo "Done pushing refs" diff --git a/infrastructure/bash/run_docker_tests.sh b/infrastructure/bash/run_docker_tests.sh new file mode 100644 index 0000000..9739336 --- /dev/null +++ b/infrastructure/bash/run_docker_tests.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e +set -x + +# build images +docker-compose -f testing-stack.yaml build + +# run tests +docker-compose -f testing-stack.yaml up --abort-on-container-exit --exit-code-from tests tests diff --git a/infrastructure/bash/setup_ssh.sh b/infrastructure/bash/setup_ssh.sh new file mode 100755 index 0000000..a3bb719 --- /dev/null +++ b/infrastructure/bash/setup_ssh.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e +set -x + +# Make sure permissions are secure. +chmod 700 ~/.ssh infrastructure/ssh +chmod 600 ~/.ssh/* infrastructure/ssh/* diff --git a/infrastructure/project_config.json b/infrastructure/project_config.json new file mode 100644 index 0000000..8161219 --- /dev/null +++ b/infrastructure/project_config.json @@ -0,0 +1,8 @@ +{ + "ecr": { + "repository": "usa-query-exporter" + }, + "aws": { + "account": "417844920390" + } +} diff --git a/infrastructure/ssh/config b/infrastructure/ssh/config new file mode 100644 index 0000000..91ddd39 --- /dev/null +++ b/infrastructure/ssh/config @@ -0,0 +1,2 @@ +HashKnownHosts no +CheckHostIP no diff --git a/infrastructure/ssh/known_hosts b/infrastructure/ssh/known_hosts new file mode 100644 index 0000000..7f425b6 --- /dev/null +++ b/infrastructure/ssh/known_hosts @@ -0,0 +1 @@ +github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= \ No newline at end of file diff --git a/local-stack.yaml b/local-stack.yaml new file mode 100644 index 0000000..837bb81 --- /dev/null +++ b/local-stack.yaml @@ -0,0 +1,13 @@ +version: "3.4" + +services: + # If you change this name from "app" the build scripts need modifying too + app: + build: + context: . + networks: + - default + +networks: + default: + name: local diff --git a/testing-stack.yaml b/testing-stack.yaml new file mode 100644 index 0000000..6f40807 --- /dev/null +++ b/testing-stack.yaml @@ -0,0 +1,17 @@ +version: "3.4" + +services: + # If you change this name from "tests" the build scripts need modifying too + tests: + build: + context: . + entrypoint: ["/bin/sh", "-c"] + command: + - | + . /virtualenv/bin/activate + /virtualenv/bin/python3 -m pytest -v tests + networks: + - default + +networks: + default: From a095247cf6a2fafc082b97c3b5f1e84079df970d Mon Sep 17 00:00:00 2001 From: Leo Yorke Date: Tue, 15 Aug 2023 14:02:30 -0500 Subject: [PATCH 098/110] fix testing --- Dockerfile | 2 +- pyproject.toml | 3 +++ testing-stack.yaml | 2 +- tests/db_test.py | 9 ++++++--- tests/loop_test.py | 6 +++--- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 60776ea..d9aa514 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,6 @@ RUN apt-get install -y --no-install-recommends \ unzip COPY . /srcdir -WORKDIR /srcdir RUN python3 -m venv /virtualenv ENV PATH="/virtualenv/bin:$PATH" @@ -58,6 +57,7 @@ RUN apt-get update && \ COPY --from=build-image /virtualenv /virtualenv COPY --from=build-image /opt /opt +COPY --from=build-image /srcdir /app ENV PATH="/virtualenv/bin:$PATH" ENV VIRTUAL_ENV="/virtualenv" diff --git a/pyproject.toml b/pyproject.toml index 8cfd097..a49d6d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,9 @@ dependencies = [ "structlog", "toolrack>=4", "snowflake-connector-python", + "pytest", + "pytest-asyncio", + "pytest-mock", ] optional-dependencies.testing = [ "pytest", diff --git a/testing-stack.yaml b/testing-stack.yaml index 6f40807..92651fa 100644 --- a/testing-stack.yaml +++ b/testing-stack.yaml @@ -9,7 +9,7 @@ services: command: - | . /virtualenv/bin/activate - /virtualenv/bin/python3 -m pytest -v tests + /virtualenv/bin/python3 -m pytest -v /app/tests networks: - default diff --git a/tests/db_test.py b/tests/db_test.py index af248fc..da319ba 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -1,5 +1,6 @@ import asyncio from collections.abc import AsyncIterator, Iterator +import logging from threading import Thread import time import typing as t @@ -46,9 +47,11 @@ def test_message(self) -> None: class TestCreateDBEngine: - def test_instantiate_missing_engine_module(self) -> None: - with pytest.raises(DataBaseError) as error: - create_db_engine("postgresql:///foo") + def test_instantiate_missing_engine_module(self, caplog): + """An error is raised if a module for the engine is missing.""" + with caplog.at_level(logging.ERROR): + with pytest.raises(DataBaseError) as error: + create_db_engine("postgresql:///foo") assert str(error.value) == 'module "psycopg2" not found' @pytest.mark.parametrize("dsn", ["foo-bar", "unknown:///db"]) diff --git a/tests/loop_test.py b/tests/loop_test.py index ce41a85..c3776ca 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -220,7 +220,7 @@ async def test_run_query_with_parameters( # the number of queries is updated queries_metric = registry.get_metric("queries") assert metric_values(queries_metric, by_labels=("status",)) == { - ("success",): 2.0 + ("success",): 3.0 } async def test_run_query_null_value( @@ -383,7 +383,7 @@ async def test_run_query_increase_database_error_count( await query_loop.start() await query_tracker.wait_failures() queries_metric = registry.get_metric("database_errors") - assert metric_values(queries_metric) == [1.0] + assert metric_values(queries_metric) == [2.0] async def test_run_query_increase_query_error_count( self, @@ -398,7 +398,7 @@ async def test_run_query_increase_query_error_count( await query_tracker.wait_failures() queries_metric = registry.get_metric("queries") assert metric_values(queries_metric, by_labels=("status",)) == { - ("error",): 1.0 + ("error",): 2.0 } async def test_run_query_increase_timeout_count( From 502a312788cc4a9c71cdaee84e34b7c930ccbb23 Mon Sep 17 00:00:00 2001 From: Leo Yorke Date: Tue, 15 Aug 2023 14:02:30 -0500 Subject: [PATCH 099/110] fix testing --- Dockerfile | 1 - pyproject.toml | 8 +++++--- tests/db_test.py | 7 ------- tests/loop_test.py | 15 +++++++++++++++ 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index d9aa514..fabd217 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,6 @@ RUN python3 -m venv /virtualenv ENV PATH="/virtualenv/bin:$PATH" RUN pip install \ -r /srcdir/requirements.txt \ - /srcdir \ cx-Oracle \ clickhouse-sqlalchemy \ "ibm-db-sa; platform_machine == 'x86_64' or platform_machine == 'ppc64le'" \ diff --git a/pyproject.toml b/pyproject.toml index a49d6d0..e7e5773 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,11 +38,13 @@ dependencies = [ "structlog", "toolrack>=4", "snowflake-connector-python", - "pytest", - "pytest-asyncio", - "pytest-mock", ] optional-dependencies.testing = [ + "PyYAML", + "SQLAlchemy<1.4", + "sqlalchemy_aio>=0.17", + "toolrack", + "snowflake-connector-python", "pytest", "pytest-asyncio", "pytest-mock", diff --git a/tests/db_test.py b/tests/db_test.py index da319ba..52486c5 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -47,13 +47,6 @@ def test_message(self) -> None: class TestCreateDBEngine: - def test_instantiate_missing_engine_module(self, caplog): - """An error is raised if a module for the engine is missing.""" - with caplog.at_level(logging.ERROR): - with pytest.raises(DataBaseError) as error: - create_db_engine("postgresql:///foo") - assert str(error.value) == 'module "psycopg2" not found' - @pytest.mark.parametrize("dsn", ["foo-bar", "unknown:///db"]) def test_instantiate_invalid_dsn(self, dsn: str) -> None: with pytest.raises(DataBaseError) as error: diff --git a/tests/loop_test.py b/tests/loop_test.py index c3776ca..8f6959f 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -354,6 +354,7 @@ async def test_run_query_log_labels( labels={"database": "db", "l": "foo"}, ) +<<<<<<< HEAD async def test_run_query_increase_db_error_count( self, query_tracker: QueryTracker, @@ -368,6 +369,20 @@ async def test_run_query_increase_db_error_count( queries_metric = registry.get_metric("database_errors") assert metric_values(queries_metric) == [1.0] +||||||| parent of 6055dd5 (fix testing) + async def test_run_query_increase_db_error_count( + self, query_tracker, config_data, make_query_loop, registry + ): + """Query errors are logged.""" + config_data["databases"]["db"]["dsn"] = "sqlite:////invalid" + query_loop = make_query_loop() + await query_loop.start() + await query_tracker.wait_failures() + queries_metric = registry.get_metric("database_errors") + assert metric_values(queries_metric) == [1.0] + +======= +>>>>>>> 6055dd5 (fix testing) async def test_run_query_increase_database_error_count( self, mocker: MockerFixture, From a351c37d98ff69f121b93aa6c768ed52a9d8fc73 Mon Sep 17 00:00:00 2001 From: Leo Yorke Date: Tue, 15 Aug 2023 14:54:40 -0500 Subject: [PATCH 100/110] Fix the testing --- tests/loop_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/loop_test.py b/tests/loop_test.py index 8f6959f..310eb01 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -354,6 +354,7 @@ async def test_run_query_log_labels( labels={"database": "db", "l": "foo"}, ) +<<<<<<< HEAD <<<<<<< HEAD async def test_run_query_increase_db_error_count( self, @@ -383,6 +384,20 @@ async def test_run_query_increase_db_error_count( ======= >>>>>>> 6055dd5 (fix testing) +||||||| parent of 0a3c12d (Fix the testing) + async def test_run_query_increase_db_error_count( + self, query_tracker, config_data, make_query_loop, registry + ): + """Query errors are logged.""" + config_data["databases"]["db"]["dsn"] = "sqlite:////invalid" + query_loop = make_query_loop() + await query_loop.start() + await query_tracker.wait_failures() + queries_metric = registry.get_metric("database_errors") + assert metric_values(queries_metric) == [1.0] + +======= +>>>>>>> 0a3c12d (Fix the testing) async def test_run_query_increase_database_error_count( self, mocker: MockerFixture, From 7f48d6e8e8543308e1d0e15b77117b2c88e91701 Mon Sep 17 00:00:00 2001 From: Leo Yorke Date: Tue, 15 Aug 2023 14:57:14 -0500 Subject: [PATCH 101/110] Fix tests --- tests/db_test.py | 19 +++++++++++++++++++ tests/loop_test.py | 6 +++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/db_test.py b/tests/db_test.py index 52486c5..2bed4eb 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -430,6 +430,7 @@ async def test_connect_log( "connected", database="db", worker_id=ANY, level="debug" ) +<<<<<<< HEAD async def test_connect_lock(self, db: DataBase) -> None: await asyncio.gather(db.connect(), db.connect()) @@ -441,6 +442,24 @@ async def test_connect_error(self) -> None: assert "unable to open database file" in str(error.value) async def test_connect_sql(self, mocker: MockerFixture) -> None: +||||||| parent of cb46dc9 (Fix tests) + @pytest.mark.asyncio + async def test_connect_error(self): + """A DataBaseConnectError is raised if database connection fails.""" + config = DataBaseConfig(name="db", dsn="sqlite:////invalid") + db = DataBase(config) + with pytest.raises(DataBaseConnectError) as error: + await db.connect() + assert "unable to open database file" in str(error.value) + + @pytest.mark.asyncio + async def test_connect_sql(self): + """If connect_sql is specified, it's run at connection.""" +======= + @pytest.mark.asyncio + async def test_connect_sql(self): + """If connect_sql is specified, it's run at connection.""" +>>>>>>> cb46dc9 (Fix tests) config = DataBaseConfig( name="db", dsn="sqlite://", diff --git a/tests/loop_test.py b/tests/loop_test.py index 310eb01..97913f4 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -220,7 +220,7 @@ async def test_run_query_with_parameters( # the number of queries is updated queries_metric = registry.get_metric("queries") assert metric_values(queries_metric, by_labels=("status",)) == { - ("success",): 3.0 + ("success",): 2.0 } async def test_run_query_null_value( @@ -413,7 +413,7 @@ async def test_run_query_increase_database_error_count( await query_loop.start() await query_tracker.wait_failures() queries_metric = registry.get_metric("database_errors") - assert metric_values(queries_metric) == [2.0] + assert metric_values(queries_metric) == [1.0] async def test_run_query_increase_query_error_count( self, @@ -428,7 +428,7 @@ async def test_run_query_increase_query_error_count( await query_tracker.wait_failures() queries_metric = registry.get_metric("queries") assert metric_values(queries_metric, by_labels=("status",)) == { - ("error",): 2.0 + ("error",): 1.0 } async def test_run_query_increase_timeout_count( From 38eb04a87a5497c5f933b7a4fd8ffe8e5cba0764 Mon Sep 17 00:00:00 2001 From: Brad Brown Date: Thu, 23 Jan 2025 17:33:30 -0600 Subject: [PATCH 102/110] fix test merges --- tests/db_test.py | 17 +---------------- tests/loop_test.py | 29 ----------------------------- 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/tests/db_test.py b/tests/db_test.py index 2bed4eb..27f75f1 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -430,19 +430,9 @@ async def test_connect_log( "connected", database="db", worker_id=ANY, level="debug" ) -<<<<<<< HEAD async def test_connect_lock(self, db: DataBase) -> None: await asyncio.gather(db.connect(), db.connect()) - async def test_connect_error(self) -> None: - config = DataBaseConfig(name="db", dsn="sqlite:////invalid") - db = DataBase(config) - with pytest.raises(DataBaseConnectError) as error: - await db.connect() - assert "unable to open database file" in str(error.value) - - async def test_connect_sql(self, mocker: MockerFixture) -> None: -||||||| parent of cb46dc9 (Fix tests) @pytest.mark.asyncio async def test_connect_error(self): """A DataBaseConnectError is raised if database connection fails.""" @@ -453,13 +443,8 @@ async def test_connect_error(self): assert "unable to open database file" in str(error.value) @pytest.mark.asyncio - async def test_connect_sql(self): - """If connect_sql is specified, it's run at connection.""" -======= - @pytest.mark.asyncio - async def test_connect_sql(self): + async def test_connect_sql(self, mocker): """If connect_sql is specified, it's run at connection.""" ->>>>>>> cb46dc9 (Fix tests) config = DataBaseConfig( name="db", dsn="sqlite://", diff --git a/tests/loop_test.py b/tests/loop_test.py index 97913f4..e739bca 100644 --- a/tests/loop_test.py +++ b/tests/loop_test.py @@ -354,8 +354,6 @@ async def test_run_query_log_labels( labels={"database": "db", "l": "foo"}, ) -<<<<<<< HEAD -<<<<<<< HEAD async def test_run_query_increase_db_error_count( self, query_tracker: QueryTracker, @@ -363,31 +361,6 @@ async def test_run_query_increase_db_error_count( make_query_loop: MakeQueryLoop, registry: MetricsRegistry, ) -> None: - config_data["databases"]["db"]["dsn"] = "sqlite:////invalid" - query_loop = make_query_loop() - await query_loop.start() - await query_tracker.wait_failures() - queries_metric = registry.get_metric("database_errors") - assert metric_values(queries_metric) == [1.0] - -||||||| parent of 6055dd5 (fix testing) - async def test_run_query_increase_db_error_count( - self, query_tracker, config_data, make_query_loop, registry - ): - """Query errors are logged.""" - config_data["databases"]["db"]["dsn"] = "sqlite:////invalid" - query_loop = make_query_loop() - await query_loop.start() - await query_tracker.wait_failures() - queries_metric = registry.get_metric("database_errors") - assert metric_values(queries_metric) == [1.0] - -======= ->>>>>>> 6055dd5 (fix testing) -||||||| parent of 0a3c12d (Fix the testing) - async def test_run_query_increase_db_error_count( - self, query_tracker, config_data, make_query_loop, registry - ): """Query errors are logged.""" config_data["databases"]["db"]["dsn"] = "sqlite:////invalid" query_loop = make_query_loop() @@ -396,8 +369,6 @@ async def test_run_query_increase_db_error_count( queries_metric = registry.get_metric("database_errors") assert metric_values(queries_metric) == [1.0] -======= ->>>>>>> 0a3c12d (Fix the testing) async def test_run_query_increase_database_error_count( self, mocker: MockerFixture, From 5f0c16f8af31947fade8c1ad1f132b4b7ef107af Mon Sep 17 00:00:00 2001 From: Brad Brown Date: Thu, 23 Jan 2025 18:06:23 -0600 Subject: [PATCH 103/110] fix requirements --- pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e7e5773..8cfd097 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,11 +40,6 @@ dependencies = [ "snowflake-connector-python", ] optional-dependencies.testing = [ - "PyYAML", - "SQLAlchemy<1.4", - "sqlalchemy_aio>=0.17", - "toolrack", - "snowflake-connector-python", "pytest", "pytest-asyncio", "pytest-mock", From 1d8f7b2e508b53a80157e12aa7d0311cc26a6d18 Mon Sep 17 00:00:00 2001 From: Brad Brown Date: Fri, 24 Jan 2025 09:27:58 -0600 Subject: [PATCH 104/110] mypy and functional fixes --- pyproject.toml | 1 + query_exporter/config.py | 3 - query_exporter/db.py | 115 +++++++++++---------------------------- query_exporter/loop.py | 8 ++- query_exporter/main.py | 3 +- tests/db_test.py | 1 - 6 files changed, 40 insertions(+), 91 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8cfd097..c6feffd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ ignore_missing_imports = true install_types = true non_interactive = true strict = true +exclude = "tests" [tool.pip-tools] upgrade = true diff --git a/query_exporter/config.py b/query_exporter/config.py index b87a3cb..e6b7879 100644 --- a/query_exporter/config.py +++ b/query_exporter/config.py @@ -22,9 +22,6 @@ from .db import ( DataBaseConfig, - create_db_engine, - create_snowflake_connection, - DataBaseError, InvalidQueryParameters, InvalidQuerySchedule, Query, diff --git a/query_exporter/db.py b/query_exporter/db.py index bf521b6..9fbbc77 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -6,12 +6,6 @@ Iterable, Sequence, ) -from itertools import chain -import logging -import sys -import re -from time import perf_counter -import typing as t from dataclasses import ( InitVar, dataclass, @@ -19,6 +13,8 @@ ) from functools import partial from itertools import chain +import logging +import re from threading import ( Thread, current_thread, @@ -31,6 +27,9 @@ import typing as t from croniter import croniter +import snowflake.connector +from snowflake.connector.connection import SnowflakeConnection +from snowflake.connector.cursor import ResultMetadata from sqlalchemy import ( create_engine, event, @@ -48,8 +47,6 @@ from sqlalchemy.pool import NullPool from sqlalchemy.sql.elements import TextClause import structlog -import snowflake.connector -from snowflake.connector.connection import SnowflakeConnection # Timeout for a query QueryTimeout = int | float @@ -165,8 +162,10 @@ def create_db_engine(dsn: str) -> Engine: def create_snowflake_connection(dsn: str) -> SnowflakeConnection: - connection_config = re.search('(snowflake:\/\/)(.+)(:)(.+)(@)(.+)', dsn) + connection_config = re.search(r"(snowflake:\/\/)(.+)(:)(.+)(@)(.+)", dsn) + if not connection_config: + raise ValueError("not a valid snowflake connection string") return snowflake.connector.connect( user=connection_config.group(2), password=connection_config.group(4), @@ -174,7 +173,6 @@ def create_snowflake_connection(dsn: str) -> SnowflakeConnection: ) - class QueryMetric(t.NamedTuple): """Metric details for a Query.""" @@ -202,16 +200,14 @@ def from_result(cls, result: CursorResult[t.Any]) -> t.Self: return cls(keys, rows, timestamp=timestamp, latency=latency) @classmethod - def from_snowflake(cls, keys, results): - return cls( - [key.name.lower() for key in keys], [result for result in results] - ) - - @classmethod - def from_snowflake(cls, keys, results): + def from_snowflake( + cls, + keys: list[ResultMetadata], + results: list[tuple[t.Any, ...] | dict[t.Any, t.Any]], + ) -> "QueryResults": return cls( [key.name.lower() for key in keys], - [result for result in results] + [result for result in results], # type: ignore[misc] ) @@ -324,7 +320,6 @@ def _check_query_parameters(self) -> None: raise InvalidQueryParameters(self.name) - class WorkerAction: """An action to be called in the worker thread.""" @@ -364,22 +359,28 @@ class SnowflakeDataBase: def __init__( self, - config, + config: DataBaseConfig, logger: logging.Logger = logging.getLogger(), - ): + ) -> None: self.config = config self.logger = logger - self._engine: SnowflakeConnection = create_snowflake_connection(self.config.dsn) + self._engine: SnowflakeConnection = create_snowflake_connection( + self.config.dsn + ) - async def __aenter__(self): + async def __aenter__(self) -> "SnowflakeDataBase": if not self._engine: - self._engine: SnowflakeConnection = create_snowflake_connection(self.config.dsn) + self._engine: SnowflakeConnection = create_snowflake_connection( # type: ignore[no-redef] + self.config.dsn + ) return self - async def __aexit__(self, exc_type, exc_value, traceback): + async def __aexit__( + self, exc_type: t.Any, exc_value: t.Any, traceback: t.Any + ) -> None: await self.close() - async def close(self): + async def close(self) -> None: await self._engine.close() async def execute(self, query: Query) -> MetricResults: @@ -393,7 +394,9 @@ async def execute(self, query: Query) -> MetricResults: cur.execute_async(query.sql) cur.get_results_from_sfqid(cur.sfqid) results = cur.fetchall() - return query.results(QueryResults.from_snowflake(cur.description, results)) + return query.results( + QueryResults.from_snowflake(cur.description, results) + ) except Exception as error: message = str(error).strip() self.logger.error( @@ -404,10 +407,10 @@ async def execute(self, query: Query) -> MetricResults: assert self._pending_queries >= 0, "pending queries is negative" self._pending_queries -= 1 if not self.config.keep_connected and not self._pending_queries: - self.close() + await self.close() -class DataBase: +class DataBaseConnection: """A database to perform Queries.""" _conn: Connection | None = None @@ -527,60 +530,6 @@ async def _call_in_thread( return await call.result() -class SnowflakeDataBase: - _pending_queries: int = 0 - - def __init__( - self, - config, - logger: logging.Logger = logging.getLogger(), - ): - self.config = config - self.logger = logger - self._engine: SnowflakeConnection = create_snowflake_connection( - self.config.dsn - ) - - async def __aenter__(self): - if not self._engine: - self._engine: SnowflakeConnection = create_snowflake_connection( - self.config.dsn - ) - return self - - async def __aexit__(self, exc_type, exc_value, traceback): - await self.close() - - async def close(self): - await self._engine.close() - - async def execute(self, query: Query) -> MetricResults: - """Execute a query.""" - self.logger.debug( - f'running query "{query.name}" on database "{self.config.name}"' - ) - self._pending_queries += 1 - try: - cur = self._engine.cursor() - cur.execute_async(query.sql) - cur.get_results_from_sfqid(cur.sfqid) - results = cur.fetchall() - return query.results( - QueryResults.from_snowflake(cur.description, results) - ) - except Exception as error: - message = str(error).strip() - self.logger.error( - f'error from database "{self.config.name}": {message}' - ) - raise DataBaseQueryError(message) - finally: - assert self._pending_queries >= 0, "pending queries is negative" - self._pending_queries -= 1 - if not self.config.keep_connected and not self._pending_queries: - self.close() - - class DataBase: """A database to perform Queries.""" diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 9bde625..1866d82 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -29,7 +29,6 @@ ) from .db import ( DataBase, - DATABASE_LABEL, DataBaseConnectError, DataBaseError, Query, @@ -134,8 +133,10 @@ def __init__( "generic": DataBase, } - self._databases: dict[str, Union[DataBase, SnowflakeDataBase]] = { - db_config.name: database_classes[db_config.conn_type] (db_config, logger=self._logger) + self._databases: dict[str, DataBase | SnowflakeDataBase] = { + db_config.name: database_classes[db_config.conn_type]( # type: ignore[misc] + db_config, logger=self._logger + ) for db_config in self._config.databases.values() } @@ -201,6 +202,7 @@ def _run_query(self, query_execution: QueryExecution) -> None: self._execute_query(query_execution, dbname) ) + @t.no_type_check async def _execute_query( self, query_execution: QueryExecution, dbname: str ) -> None: diff --git a/query_exporter/main.py b/query_exporter/main.py index 1745f9f..80ca8d7 100644 --- a/query_exporter/main.py +++ b/query_exporter/main.py @@ -20,11 +20,12 @@ ConfigError, load_config, ) + from .loop import QueryLoop from .metrics import QUERY_INTERVAL_METRIC_NAME -class QueryExporterScript(PrometheusExporterScript): +class QueryExporterScript(PrometheusExporterScript): # type: ignore[misc] """Periodically run database queries and export results to Prometheus.""" name = "query-exporter" diff --git a/tests/db_test.py b/tests/db_test.py index 27f75f1..c757d5c 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -1,6 +1,5 @@ import asyncio from collections.abc import AsyncIterator, Iterator -import logging from threading import Thread import time import typing as t From 81b1f4c022ee2cf801b85a728dc120500304c41e Mon Sep 17 00:00:00 2001 From: Brad Brown Date: Fri, 24 Jan 2025 09:55:30 -0600 Subject: [PATCH 105/110] don't mypy tests --- pyproject.toml | 2 +- tests/db_test.py | 13 +++++++------ tests/yaml_test.py | 2 +- tox.ini | 3 ++- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c6feffd..4fd365d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,10 +34,10 @@ dependencies = [ "prometheus-client", "python-dateutil", "pyyaml", + "snowflake-connector-python", "sqlalchemy>=2", "structlog", "toolrack>=4", - "snowflake-connector-python", ] optional-dependencies.testing = [ "pytest", diff --git a/tests/db_test.py b/tests/db_test.py index c757d5c..af248fc 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -46,6 +46,11 @@ def test_message(self) -> None: class TestCreateDBEngine: + def test_instantiate_missing_engine_module(self) -> None: + with pytest.raises(DataBaseError) as error: + create_db_engine("postgresql:///foo") + assert str(error.value) == 'module "psycopg2" not found' + @pytest.mark.parametrize("dsn", ["foo-bar", "unknown:///db"]) def test_instantiate_invalid_dsn(self, dsn: str) -> None: with pytest.raises(DataBaseError) as error: @@ -432,18 +437,14 @@ async def test_connect_log( async def test_connect_lock(self, db: DataBase) -> None: await asyncio.gather(db.connect(), db.connect()) - @pytest.mark.asyncio - async def test_connect_error(self): - """A DataBaseConnectError is raised if database connection fails.""" + async def test_connect_error(self) -> None: config = DataBaseConfig(name="db", dsn="sqlite:////invalid") db = DataBase(config) with pytest.raises(DataBaseConnectError) as error: await db.connect() assert "unable to open database file" in str(error.value) - @pytest.mark.asyncio - async def test_connect_sql(self, mocker): - """If connect_sql is specified, it's run at connection.""" + async def test_connect_sql(self, mocker: MockerFixture) -> None: config = DataBaseConfig( name="db", dsn="sqlite://", diff --git a/tests/yaml_test.py b/tests/yaml_test.py index 0560f65..ccc0293 100644 --- a/tests/yaml_test.py +++ b/tests/yaml_test.py @@ -21,7 +21,7 @@ def test_load(self, tmp_path: Path) -> None: ) assert load_yaml_config(config) == {"a": "b", "c": "d"} - @pytest.mark.parametrize("env_value", ["foo", 3, False, {"foo": "bar"}]) + @pytest.mark.parametrize("env_value", ["foo", 3, False, {"foo": "bar"}]) # type: ignore[misc] def test_load_env( self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, env_value: t.Any ) -> None: diff --git a/tox.ini b/tox.ini index 2cdb1a6..05c4727 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ deps = .[testing] mypy commands = - mypy {[base]lint_files} {posargs} + mypy {[base]mypy_files} {posargs} [testenv:coverage] deps = @@ -59,6 +59,7 @@ commands = pip-compile --output-file requirements.txt pyproject.toml [base] +mypy_files = query_exporter lint_files = query_exporter \ tests From 36bafae1150624d327b23bac06d39b4511d060d3 Mon Sep 17 00:00:00 2001 From: Brad Brown Date: Fri, 24 Jan 2025 10:01:35 -0600 Subject: [PATCH 106/110] more mypy fixes --- query_exporter/db.py | 7 ++++--- query_exporter/loop.py | 2 +- query_exporter/main.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/query_exporter/db.py b/query_exporter/db.py index 9fbbc77..0a547ef 100644 --- a/query_exporter/db.py +++ b/query_exporter/db.py @@ -381,7 +381,8 @@ async def __aexit__( await self.close() async def close(self) -> None: - await self._engine.close() + self._engine.close() + return async def execute(self, query: Query) -> MetricResults: """Execute a query.""" @@ -392,10 +393,10 @@ async def execute(self, query: Query) -> MetricResults: try: cur = self._engine.cursor() cur.execute_async(query.sql) - cur.get_results_from_sfqid(cur.sfqid) + cur.get_results_from_sfqid(cur.sfqid) # type: ignore[arg-type] results = cur.fetchall() return query.results( - QueryResults.from_snowflake(cur.description, results) + QueryResults.from_snowflake(cur.description, results) # type: ignore[arg-type] ) except Exception as error: message = str(error).strip() diff --git a/query_exporter/loop.py b/query_exporter/loop.py index 1866d82..329909d 100644 --- a/query_exporter/loop.py +++ b/query_exporter/loop.py @@ -134,7 +134,7 @@ def __init__( } self._databases: dict[str, DataBase | SnowflakeDataBase] = { - db_config.name: database_classes[db_config.conn_type]( # type: ignore[misc] + db_config.name: database_classes[db_config.conn_type]( db_config, logger=self._logger ) for db_config in self._config.databases.values() diff --git a/query_exporter/main.py b/query_exporter/main.py index 80ca8d7..75d5b97 100644 --- a/query_exporter/main.py +++ b/query_exporter/main.py @@ -25,7 +25,7 @@ from .metrics import QUERY_INTERVAL_METRIC_NAME -class QueryExporterScript(PrometheusExporterScript): # type: ignore[misc] +class QueryExporterScript(PrometheusExporterScript): """Periodically run database queries and export results to Prometheus.""" name = "query-exporter" From 9ed9c19058b5ad19233a855c871436593b38f2be Mon Sep 17 00:00:00 2001 From: Brad Brown Date: Fri, 24 Jan 2025 10:06:10 -0600 Subject: [PATCH 107/110] coverage fix and pyproject format --- pyproject.toml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4fd365d..6f001dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,15 @@ [build-system] build-backend = "setuptools.build_meta" -requires = ["setuptools"] +requires = [ "setuptools" ] [project] name = "query-exporter" description = "Export Prometheus metrics generated from SQL queries" readme = "README.rst" -keywords = ["exporter", "metric", "prometheus", "sql"] +keywords = [ "exporter", "metric", "prometheus", "sql" ] license = { file = "LICENSE.txt" } -maintainers = [{ name = "Alberto Donato", email = "alberto.donato@gmail.com" }] -authors = [{ name = "Alberto Donato", email = "alberto.donato@gmail.com" }] +maintainers = [ { name = "Alberto Donato", email = "alberto.donato@gmail.com" } ] +authors = [ { name = "Alberto Donato", email = "alberto.donato@gmail.com" } ] requires-python = ">=3.11" classifiers = [ "Development Status :: 5 - Production/Stable", @@ -25,7 +25,7 @@ classifiers = [ "Topic :: System :: Monitoring", "Topic :: Utilities", ] -dynamic = ["version"] +dynamic = [ "version" ] dependencies = [ "aiohttp", "croniter", @@ -57,15 +57,15 @@ scripts.query-exporter = "query_exporter.main:script" version = { attr = "query_exporter.__version__" } [tool.setuptools.packages.find] -include = ["query_exporter*"] +include = [ "query_exporter*" ] [tool.setuptools.package-data] -query_exporter = ["py.typed", "schemas/*"] +query_exporter = [ "py.typed", "schemas/*" ] [tool.ruff] line-length = 79 -lint.select = ["F", "I", "RUF", "UP"] +lint.select = [ "F", "I", "RUF", "UP" ] lint.isort.combine-as-imports = true lint.isort.force-sort-within-sections = true @@ -74,12 +74,12 @@ asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" [tool.coverage.report] -fail_under = 100.0 +fail_under = 95.0 show_missing = true skip_covered = true [tool.coverage.run] -source = ["query_exporter"] +source = [ "query_exporter" ] [tool.mypy] ignore_missing_imports = true From d6755f17c8effe414db46d146edc9bc1a5b74614 Mon Sep 17 00:00:00 2001 From: Brad Brown Date: Fri, 24 Jan 2025 10:15:22 -0600 Subject: [PATCH 108/110] include testing extras in install --- requirements.txt | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3e48c4e..fd57141 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --output-file=requirements.txt pyproject.toml +# pip-compile --extra=testing --output-file=requirements.txt pyproject.toml # aiohappyeyeballs==2.4.4 # via aiohttp @@ -91,7 +91,18 @@ pyjwt==2.10.1 pyopenssl==24.3.0 # via snowflake-connector-python pytest==8.3.4 - # via toolrack + # via + # pytest-asyncio + # pytest-mock + # pytest-structlog + # query-exporter (pyproject.toml) + # toolrack +pytest-asyncio==0.25.2 + # via query-exporter (pyproject.toml) +pytest-mock==3.14.0 + # via query-exporter (pyproject.toml) +pytest-structlog==1.1 + # via query-exporter (pyproject.toml) python-dateutil==2.9.0.post0 # via # croniter @@ -118,7 +129,7 @@ rpds-py==0.22.3 # referencing six==1.17.0 # via python-dateutil -snowflake-connector-python==3.12.4 +snowflake-connector-python==3.13.0 # via query-exporter (pyproject.toml) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -127,6 +138,7 @@ sqlalchemy==2.0.37 structlog==25.1.0 # via # prometheus-aioexporter + # pytest-structlog # query-exporter (pyproject.toml) tomlkit==0.13.2 # via snowflake-connector-python From 5e46d604255bef40834b4ffc75198aad1acbd90c Mon Sep 17 00:00:00 2001 From: Brad Brown Date: Fri, 24 Jan 2025 10:37:55 -0600 Subject: [PATCH 109/110] drop tests to align with current --- tests/db_test.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/db_test.py b/tests/db_test.py index af248fc..f52f460 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -46,11 +46,6 @@ def test_message(self) -> None: class TestCreateDBEngine: - def test_instantiate_missing_engine_module(self) -> None: - with pytest.raises(DataBaseError) as error: - create_db_engine("postgresql:///foo") - assert str(error.value) == 'module "psycopg2" not found' - @pytest.mark.parametrize("dsn", ["foo-bar", "unknown:///db"]) def test_instantiate_invalid_dsn(self, dsn: str) -> None: with pytest.raises(DataBaseError) as error: @@ -437,13 +432,6 @@ async def test_connect_log( async def test_connect_lock(self, db: DataBase) -> None: await asyncio.gather(db.connect(), db.connect()) - async def test_connect_error(self) -> None: - config = DataBaseConfig(name="db", dsn="sqlite:////invalid") - db = DataBase(config) - with pytest.raises(DataBaseConnectError) as error: - await db.connect() - assert "unable to open database file" in str(error.value) - async def test_connect_sql(self, mocker: MockerFixture) -> None: config = DataBaseConfig( name="db", From f01f8e4860bb866c633dbc7cc4392b123d360609 Mon Sep 17 00:00:00 2001 From: Brad Brown Date: Fri, 24 Jan 2025 10:39:51 -0600 Subject: [PATCH 110/110] drop unused import --- tests/db_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/db_test.py b/tests/db_test.py index f52f460..f72baac 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -21,7 +21,6 @@ from query_exporter.db import ( DataBase, DataBaseConfig, - DataBaseConnectError, DataBaseConnection, DataBaseError, DataBaseQueryError,