diff --git a/README.md b/README.md index 56a6366..3504244 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ You can modify environment variables to change the behavior of the exporter. | `PREFECT_CSRF_ENABLED` | Enable compatibilty with Prefect Servers using CSRF protection | `False` | | `PAGINATION_ENABLED` | Enable pagination usage. (Uses more resources) | `True` | | `PAGINATION_LIMIT` | Pagination limit | `200` | +| `SCRAPE_INTERVAL_SECONDS` | Interval in seconds to scrape metrics from Prefect API | `30` | ## Contributing diff --git a/main.py b/main.py index e6092cf..13b3e19 100644 --- a/main.py +++ b/main.py @@ -17,10 +17,11 @@ def metrics(): loglevel = str(os.getenv("LOG_LEVEL", "INFO")) max_retries = int(os.getenv("MAX_RETRIES", "3")) metrics_port = int(os.getenv("METRICS_PORT", "8000")) - offset_minutes = int(os.getenv("OFFSET_MINUTES", "5")) + offset_minutes = int(os.getenv("OFFSET_MINUTES", "3")) url = str(os.getenv("PREFECT_API_URL", "http://localhost:4200/api")) api_key = str(os.getenv("PREFECT_API_KEY", "")) csrf_client_id = str(uuid.uuid4()) + scrape_interval_seconds = int(os.getenv("SCRAPE_INTERVAL_SECONDS", "30")) # Configure logging logging.basicConfig( level=loglevel, format="%(asctime)s - %(name)s - [%(levelname)s] %(message)s" @@ -73,7 +74,7 @@ def metrics(): # Run the loop to collect Prefect metrics while True: - time.sleep(5) + time.sleep(scrape_interval_seconds) if __name__ == "__main__": diff --git a/metrics/flow_runs.py b/metrics/flow_runs.py index d573364..e4b292c 100644 --- a/metrics/flow_runs.py +++ b/metrics/flow_runs.py @@ -71,6 +71,13 @@ def get_all_flow_runs_info(self) -> list: Returns: dict: JSON response containing flow runs information. """ - all_flow_runs = self._get_with_pagination() + all_flow_runs = self._get_with_pagination( + base_data={ + "flow_runs": { + "operator": "and_", + "end_time": {"after_": f"{self.after_data_fmt}"}, + } + } + ) return all_flow_runs diff --git a/metrics/metrics.py b/metrics/metrics.py index 7dd9ff0..df7bf9f 100644 --- a/metrics/metrics.py +++ b/metrics/metrics.py @@ -36,8 +36,12 @@ def __init__( offset_minutes (int): Time offset in minutes. max_retries (int): The maximum number of retries for HTTP requests. logger (obj): The logger object. - + csrf_enabled (bool): Whether CSRF is enabled. + client_id (str): The client ID for CSRF. + enable_pagination (bool): Whether pagination is enabled. + pagination_limit (int): The pagination limit. """ + self.headers = headers self.offset_minutes = offset_minutes self.url = url