Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add computation job queue queries #362

Merged
merged 2 commits into from
Oct 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 0 additions & 40 deletions .github/workflows/pylinting.yml

This file was deleted.

3 changes: 3 additions & 0 deletions pyproject.toml.unversioned
Original file line number Diff line number Diff line change
Expand Up @@ -191,13 +191,16 @@ packages = [
"delete_feature.py",
"upload_sync_small.py",
"collection.py",
"load_query.py",
]
"spatialprofilingtoolbox.db.data_model" = [
"metaschema.sql",
"drop_metaschema.sql",
"create_roles.sql",
"create_views.sql",
"drop_views.sql",
"load_query.sql",
"load_query_breakdown.sql",
"fields.tsv",
"grant_on_tables.sql",
"performance_tweaks.sql",
Expand Down
21 changes: 21 additions & 0 deletions spatialprofilingtoolbox/db/data_model/load_query.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
SELECT
SUM(
(
xpath(
'/row/c/text()',
query_to_xml(
format(
'select count(*) as c from %I.%I',
schema_name,
'quantitative_feature_value_queue'
),
FALSE,
TRUE,
''
)
)
)[1]::text::int
) as total_jobs_in_queue
FROM
default_study_lookup.study_lookup
;
20 changes: 20 additions & 0 deletions spatialprofilingtoolbox/db/data_model/load_query_breakdown.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
SELECT
schema_name,
(
xpath(
'/row/c/text()',
query_to_xml(
format(
'select count(*) as c from %I.%I',
schema_name,
'quantitative_feature_value_queue'
),
FALSE,
TRUE,
''
)
)
)[1]::text::int AS number_jobs_in_queue
FROM
default_study_lookup.study_lookup
;
53 changes: 53 additions & 0 deletions spatialprofilingtoolbox/db/scripts/load_query.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
"""Utility to print the SQL query that returns the number of computation jobs in the queue."""

import argparse
from importlib.resources import as_file
from importlib.resources import files


def _get_data_model_file(filename: str) -> str:
source_package = 'spatialprofilingtoolbox.db.data_model'
with as_file(files(source_package).joinpath(filename)) as path:
with open(path, encoding='utf-8') as file:
script = file.read()
return script

def get_load_query() -> str:
return _get_data_model_file('load_query.sql')


def get_load_query_breakdown() -> str:
return _get_data_model_file('load_query_breakdown.sql')


def parse_args():
parser = argparse.ArgumentParser(
prog='spt db load-query',
description='Get a SQL query which returns the size of the computation job queue, suitable as a measure of load.',
)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
'--all',
action='store_true',
default=False,
help='If selected, the SQL query will be the one which sums all job queue sizes for a single total.',
)
group.add_argument(
'--breakdown-by-dataset',
action='store_true',
default=False,
help='If selected, the SQL query will return the job queue sizes broken down by dataset.',
)
return parser.parse_args()


def main():
args = parse_args()
if args.all:
print(get_load_query())
if args.breakdown_by_dataset:
print(get_load_query_breakdown())


if __name__=='__main__':
main()