From 8f20acb9c4c588e4f0160acfef0530236763a097 Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Wed, 10 Jul 2024 12:37:55 -0700 Subject: [PATCH] Serialize worker parameters before dispatching Using pickle to serialize parameters for the workers runs the risk of not being able to unpickle in a backend using a different Python version and different library versions. Reduce the risk by always serializing the parameters model before dispatching, so what will be pickled is only a JSON-compatible dictionary, which should be safely portable. Making this work properly required fixing a few bugs in the code to deserialize astropy data types since JSON serialization turns tuples into lists. Unpin numpy, since that pin should no longer be necessary. --- requirements/main.in | 4 -- requirements/main.txt | 84 +++++++++++++++------------ requirements/tox.txt | 38 ++++++------ src/vocutouts/models/domain/cutout.py | 8 +-- src/vocutouts/uws/service.py | 2 +- src/vocutouts/uws/uwsworker.py | 12 ++-- src/vocutouts/workers/cutout.py | 1 + tests/handlers/async_test.py | 3 +- tests/models/domain/cutout_test.py | 52 ++++++----------- tests/uws/job_api_test.py | 4 +- tests/uws/workers_test.py | 1 + 11 files changed, 101 insertions(+), 108 deletions(-) diff --git a/requirements/main.in b/requirements/main.in index d9de486..fb8c652 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -25,10 +25,6 @@ safir[arq,db,gcs]>=6.0.0 sqlalchemy[asyncio] structlog -# Temporarily pin numpy to version 1 so that numpy data types picked in the -# frontend can be unpickled in the backend with a current pipelines container. -numpy<2 - # Uncomment this, change the branch, comment out safir above, and run make # update-deps-no-hashes to test against an unreleased version of Safir. #safir[arq,db,gcs] @ git+https://github.com/lsst-sqre/safir@main diff --git a/requirements/main.txt b/requirements/main.txt index 7e16e58..5896606 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -725,45 +725,53 @@ mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via markdown-it-py -numpy==1.26.4 \ - --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ - --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ - --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ - --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ - --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ - --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ - --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ - --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ - --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ - --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ - --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ - --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ - --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ - --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ - --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ - --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ - --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ - --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ - --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ - --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ - --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ - --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ - --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ - --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ - --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ - --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ - --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ - --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ - --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ - --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ - --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ - --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ - --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ - --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ - --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ - --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f +numpy==2.0.0 \ + --hash=sha256:04494f6ec467ccb5369d1808570ae55f6ed9b5809d7f035059000a37b8d7e86f \ + --hash=sha256:0a43f0974d501842866cc83471bdb0116ba0dffdbaac33ec05e6afed5b615238 \ + --hash=sha256:0e50842b2295ba8414c8c1d9d957083d5dfe9e16828b37de883f51fc53c4016f \ + --hash=sha256:0ec84b9ba0654f3b962802edc91424331f423dcf5d5f926676e0150789cb3d95 \ + --hash=sha256:17067d097ed036636fa79f6a869ac26df7db1ba22039d962422506640314933a \ + --hash=sha256:1cde1753efe513705a0c6d28f5884e22bdc30438bf0085c5c486cdaff40cd67a \ + --hash=sha256:1e72728e7501a450288fc8e1f9ebc73d90cfd4671ebbd631f3e7857c39bd16f2 \ + --hash=sha256:2635dbd200c2d6faf2ef9a0d04f0ecc6b13b3cad54f7c67c61155138835515d2 \ + --hash=sha256:2ce46fd0b8a0c947ae047d222f7136fc4d55538741373107574271bc00e20e8f \ + --hash=sha256:34f003cb88b1ba38cb9a9a4a3161c1604973d7f9d5552c38bc2f04f829536609 \ + --hash=sha256:354f373279768fa5a584bac997de6a6c9bc535c482592d7a813bb0c09be6c76f \ + --hash=sha256:38ecb5b0582cd125f67a629072fed6f83562d9dd04d7e03256c9829bdec027ad \ + --hash=sha256:3e8e01233d57639b2e30966c63d36fcea099d17c53bf424d77f088b0f4babd86 \ + --hash=sha256:3f6bed7f840d44c08ebdb73b1825282b801799e325bcbdfa6bc5c370e5aecc65 \ + --hash=sha256:4554eb96f0fd263041baf16cf0881b3f5dafae7a59b1049acb9540c4d57bc8cb \ + --hash=sha256:46e161722e0f619749d1cd892167039015b2c2817296104487cd03ed4a955995 \ + --hash=sha256:49d9f7d256fbc804391a7f72d4a617302b1afac1112fac19b6c6cec63fe7fe8a \ + --hash=sha256:4d2f62e55a4cd9c58c1d9a1c9edaedcd857a73cb6fda875bf79093f9d9086f85 \ + --hash=sha256:5f64641b42b2429f56ee08b4f427a4d2daf916ec59686061de751a55aafa22e4 \ + --hash=sha256:63b92c512d9dbcc37f9d81b123dec99fdb318ba38c8059afc78086fe73820275 \ + --hash=sha256:6d7696c615765091cc5093f76fd1fa069870304beaccfd58b5dcc69e55ef49c1 \ + --hash=sha256:79e843d186c8fb1b102bef3e2bc35ef81160ffef3194646a7fdd6a73c6b97196 \ + --hash=sha256:821eedb7165ead9eebdb569986968b541f9908979c2da8a4967ecac4439bae3d \ + --hash=sha256:84554fc53daa8f6abf8e8a66e076aff6ece62de68523d9f665f32d2fc50fd66e \ + --hash=sha256:8d83bb187fb647643bd56e1ae43f273c7f4dbcdf94550d7938cfc32566756514 \ + --hash=sha256:903703372d46bce88b6920a0cd86c3ad82dae2dbef157b5fc01b70ea1cfc430f \ + --hash=sha256:9416a5c2e92ace094e9f0082c5fd473502c91651fb896bc17690d6fc475128d6 \ + --hash=sha256:9a1712c015831da583b21c5bfe15e8684137097969c6d22e8316ba66b5baabe4 \ + --hash=sha256:9c27f0946a3536403efb0e1c28def1ae6730a72cd0d5878db38824855e3afc44 \ + --hash=sha256:a356364941fb0593bb899a1076b92dfa2029f6f5b8ba88a14fd0984aaf76d0df \ + --hash=sha256:a7039a136017eaa92c1848152827e1424701532ca8e8967fe480fe1569dae581 \ + --hash=sha256:acd3a644e4807e73b4e1867b769fbf1ce8c5d80e7caaef0d90dcdc640dfc9787 \ + --hash=sha256:ad0c86f3455fbd0de6c31a3056eb822fc939f81b1618f10ff3406971893b62a5 \ + --hash=sha256:b4c76e3d4c56f145d41b7b6751255feefae92edbc9a61e1758a98204200f30fc \ + --hash=sha256:b6f6a8f45d0313db07d6d1d37bd0b112f887e1369758a5419c0370ba915b3871 \ + --hash=sha256:c5a59996dc61835133b56a32ebe4ef3740ea5bc19b3983ac60cc32be5a665d54 \ + --hash=sha256:c73aafd1afca80afecb22718f8700b40ac7cab927b8abab3c3e337d70e10e5a2 \ + --hash=sha256:cee6cc0584f71adefe2c908856ccc98702baf95ff80092e4ca46061538a2ba98 \ + --hash=sha256:cef04d068f5fb0518a77857953193b6bb94809a806bd0a14983a8f12ada060c9 \ + --hash=sha256:cf5d1c9e6837f8af9f92b6bd3e86d513cdc11f60fd62185cc49ec7d1aba34864 \ + --hash=sha256:e61155fae27570692ad1d327e81c6cf27d535a5d7ef97648a17d922224b216de \ + --hash=sha256:e7f387600d424f91576af20518334df3d97bc76a300a755f9a8d6e4f5cadd289 \ + --hash=sha256:ed08d2703b5972ec736451b818c2eb9da80d66c3e84aed1deeb0c345fefe461b \ + --hash=sha256:fbd6acc766814ea6443628f4e6751d0da6593dae29c08c0b2606164db026970c \ + --hash=sha256:feff59f27338135776f6d4e2ec7aeeac5d5f7a08a83e80869121ef8164b74af9 # via - # -r requirements/main.in # astropy # pyerfa orjson==3.10.6 \ diff --git a/requirements/tox.txt b/requirements/tox.txt index 951d40d..c39ef27 100644 --- a/requirements/tox.txt +++ b/requirements/tox.txt @@ -192,25 +192,25 @@ urllib3==2.2.2 \ # -c requirements/main.txt # docker # requests -uv==0.2.23 \ - --hash=sha256:052d277d950d6a0a9d7a4e5d1b49ee29a5c2f421cff579e81c0b15bf77b16f53 \ - --hash=sha256:09b1e7c082c92842c318d2e751388bfb34c4b39d5e53bddfae68782276996011 \ - --hash=sha256:0fadea0f534c2296a91c6624fb48a65343d3781f879271667624882e5e7fbb54 \ - --hash=sha256:162de3ec4c462611ae273b05967ef518cd488d049b4673bfa76160fb77b6f22f \ - --hash=sha256:3e938c7ab240abf9b8aa692976a044bdbf961917bf336f2b793ef219e723a78c \ - --hash=sha256:4b53383cf3a8d10c73037360591eeff458d9181ba0fa812895313ff73b62a556 \ - --hash=sha256:4b93b30ff8a416d317071bf9ce85fd2a8c86b984dbed44e0092bd6935c68fab4 \ - --hash=sha256:509b5b9c527e7e528838088076b745cd306796f65b562a1d891cf17e5e737359 \ - --hash=sha256:55b028cf0966b14317a27cf959967c52d96fdad39ed4eb8d00c5b9c0876d6c94 \ - --hash=sha256:607ac512ae22dd57de531f3d5ee4066cb96a16f2e13475326dff30b8c3105012 \ - --hash=sha256:662a79e530b80e69224aff312f9aa15172e4b3b8f2e4b5f812ff8b3eb0fa64a3 \ - --hash=sha256:693160ebe961510d5c59599fa5523fede124a0e6be3bb569dd0dce3678f1138a \ - --hash=sha256:720de561d24bd71b86811d17c7e98349991b6e93cdbc7a1d889155ee789a2a77 \ - --hash=sha256:81012dbe5cb1960a3a2359118b07d6695a1f157951320bf938d0e2b04d78f75c \ - --hash=sha256:93f10122ac98844a1ab1c455b7db2df5564bc277f4d6c414129fef109c8a8b6f \ - --hash=sha256:961272f76d56ec2e0c6c95466fc2e4b1a16e8abe75b378c87d4fc0ba0305f7c3 \ - --hash=sha256:ec58983a7c161cf759de6a0a5fba64aa9b4aa013f877fc722e057a5244cc93c2 \ - --hash=sha256:f032ac895c42dad2c7950016590a4c57e25b40f83ae84cc157f6d86fcdcb37de +uv==0.2.24 \ + --hash=sha256:06b95969d73c804bbee63fb1bde54d1a7570826b446a2b5b9136534ebf55a168 \ + --hash=sha256:0ea783d55d10bd3cf619fc1a7cb3f14e3157ffaa578a6afa068d0257923a31a4 \ + --hash=sha256:1569430e2051bb2ed80567718e81a6660407eb02a07157d648ee570fe67625e3 \ + --hash=sha256:157b5cbda857d0d88469ab86c0d4a809826b6730fce14a53b2641f75ab92768f \ + --hash=sha256:1cdde9c56ec74cf7446b1ee9668062c11755b5098dc41c5650bf50ef4fe90407 \ + --hash=sha256:1d35459408a9d4d251b03eacc7e17a6398953cae20d6275b1dca9cdb470ee992 \ + --hash=sha256:2c9bc248d3e92d6dad77c19c9aa8fa6e83d7e3b43c4fb321e28f19dba8d82c3a \ + --hash=sha256:32abd6dd5764e27402a4444dac06ee84593bf72ea1986cdd869a57283655dfcd \ + --hash=sha256:38c7614dc957b31c2064f0cd0d966805ad4af12a181dd67ebafb7c523101cf40 \ + --hash=sha256:5ef97f91bc8f51d62693ec863f923ffa3b7c9f2b0d8e365751e51a8eb7d9c8c1 \ + --hash=sha256:6176236d10b08d0e837d4a151e956a6af0e96626f0eda309888aa7809dc3c0d7 \ + --hash=sha256:7a98635a4a40eadef0fe8eb649f1451f65b77b7649503034ca4af669eda43b75 \ + --hash=sha256:8e1858e540ceb1615b35e6b85bbf429a3875d17d240bcd3bb5305406063a641d \ + --hash=sha256:c95116f2761488e4a270ab196978440319c673ecefa0db13019612861e28f395 \ + --hash=sha256:e27d3dde20bd74fc2feaa57f728b3d6c6918fd109761de88f1ffe95ed340b985 \ + --hash=sha256:e8c89ee50def595bd0461d009b91fc897b6ea94f1db3d67fa223154784b11a02 \ + --hash=sha256:f6cc8ed15d8399cfdb83a723a7971ce60ef18497198df54d104ae263a98940a4 \ + --hash=sha256:f9de35cfa2c2c4e7e041442173fbf660a15e2ede10c7f8f038b8cd9b389d2010 # via tox-uv virtualenv==20.26.3 \ --hash=sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a \ diff --git a/src/vocutouts/models/domain/cutout.py b/src/vocutouts/models/domain/cutout.py index ce9d199..cae5c46 100644 --- a/src/vocutouts/models/domain/cutout.py +++ b/src/vocutouts/models/domain/cutout.py @@ -10,7 +10,7 @@ from __future__ import annotations -from typing import Annotated, Literal, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias from astropy import units as u from astropy.coordinates import Angle, SkyCoord @@ -48,12 +48,10 @@ def _serialize_sky_coord( return (float(c.ra.degree), float(c.dec.degree)) -def _deserialize_sky_coord( - c: SkyCoord | tuple[float, float] | list[tuple[float, float]], -) -> SkyCoord: +def _deserialize_sky_coord(c: Any) -> SkyCoord: if isinstance(c, SkyCoord): return c - elif isinstance(c, list): + if isiterable(c[0]): ras = [v[0] for v in c] decs = [v[1] for v in c] return SkyCoord(ras * u.degree, decs * u.degree, frame="icrs") diff --git a/src/vocutouts/uws/service.py b/src/vocutouts/uws/service.py index 2a58279..bf7d577 100644 --- a/src/vocutouts/uws/service.py +++ b/src/vocutouts/uws/service.py @@ -344,7 +344,7 @@ async def start(self, user: str, job_id: str, token: str) -> JobMetadata: timeout=job.execution_duration, run_id=job.run_id, ) - params = params_model.to_worker_parameters() + params = params_model.to_worker_parameters().model_dump(mode="json") metadata = await self._arq.enqueue(self._config.worker, params, info) await self._storage.mark_queued(job_id, metadata) logger.info("Started job", arq_job_id=metadata.id) diff --git a/src/vocutouts/uws/uwsworker.py b/src/vocutouts/uws/uwsworker.py index 1cc6669..ba77b9a 100644 --- a/src/vocutouts/uws/uwsworker.py +++ b/src/vocutouts/uws/uwsworker.py @@ -10,7 +10,7 @@ from datetime import UTC, datetime, timedelta from enum import Enum from traceback import format_exception -from typing import Any, ClassVar, TypeVar +from typing import Any, ClassVar, Generic, TypeVar from urllib.parse import urlsplit from arq import func @@ -42,7 +42,7 @@ @dataclass -class WorkerConfig: +class WorkerConfig(Generic[T]): """Minimal configuration needed for building a UWS backend worker.""" arq_mode: ArqMode @@ -54,6 +54,9 @@ class WorkerConfig: arq_queue_password: str | None """Password of the Redis arq queue.""" + parameters_class: type[T] + """Class of the parameters to pass to the backend worker.""" + timeout: timedelta """Maximum execution time. @@ -270,7 +273,7 @@ class WorkerUsageError(WorkerError): def build_worker( worker: Callable[[T, WorkerJobInfo, BoundLogger], list[WorkerResult]], - config: WorkerConfig, + config: WorkerConfig[T], logger: BoundLogger, ) -> WorkerSettings: """Construct an arq worker for the provided backend function. @@ -346,12 +349,13 @@ async def shutdown(ctx: dict[Any, Any]) -> None: logger.info("Worker shutdown complete") async def run( - ctx: dict[Any, Any], params: T, info: WorkerJobInfo + ctx: dict[Any, Any], params_raw: dict[str, Any], info: WorkerJobInfo ) -> list[WorkerResult]: arq: ArqQueue = ctx["arq"] logger: BoundLogger = ctx["logger"] pool: ThreadPoolExecutor = ctx["pool"] + params = config.parameters_class.model_validate(params_raw) logger = logger.bind( task=worker.__qualname__, job_id=info.job_id, diff --git a/src/vocutouts/workers/cutout.py b/src/vocutouts/workers/cutout.py index 02b0a1b..4649098 100644 --- a/src/vocutouts/workers/cutout.py +++ b/src/vocutouts/workers/cutout.py @@ -201,6 +201,7 @@ def cutout( arq_mode=ArqMode.production, arq_queue_url=os.environ["CUTOUT_ARQ_QUEUE_URL"], arq_queue_password=os.getenv("CUTOUT_ARQ_QUEUE_PASSWORD"), + parameters_class=WorkerCutout, timeout=timedelta(seconds=int(os.environ["CUTOUT_TIMEOUT"])), ), structlog.get_logger("vocutouts"), diff --git a/tests/handlers/async_test.py b/tests/handlers/async_test.py index f6bde91..bd9349a 100644 --- a/tests/handlers/async_test.py +++ b/tests/handlers/async_test.py @@ -98,7 +98,8 @@ async def test_create_job(client: AsyncClient, runner: MockJobRunner) -> None: async def run_job() -> None: arq_job = await runner.get_job_metadata("someone", "2") - assert isinstance(arq_job.args[0], WorkerCutout) + assert isinstance(arq_job.args[0], dict) + assert WorkerCutout.model_validate(arq_job.args[0]) await runner.mark_in_progress("someone", "2", delay=0.2) results = [ UWSJobResult( diff --git a/tests/models/domain/cutout_test.py b/tests/models/domain/cutout_test.py index 020c2ca..af00df5 100644 --- a/tests/models/domain/cutout_test.py +++ b/tests/models/domain/cutout_test.py @@ -3,7 +3,6 @@ from __future__ import annotations import math -import pickle from vocutouts.models.cutout import ( CircleStencil, @@ -14,47 +13,31 @@ from vocutouts.models.domain.cutout import WorkerCutout, WorkerPolygonStencil -def test_pickle() -> None: - for cutout in ( - CutoutParameters( - ids=["foo"], - stencils=[CircleStencil.from_string("1 1.42 1")], - ).to_worker_parameters(), - CutoutParameters( - ids=["foo"], - stencils=[PolygonStencil.from_string("1 0 1 1 0 1 0 0")], - ).to_worker_parameters(), - ): - cutout_pickle = pickle.loads(pickle.dumps(cutout)) - assert cutout.dataset_ids == cutout_pickle.dataset_ids - expected = [s.model_dump(mode="json") for s in cutout.stencils] - seen = [s.model_dump(mode="json") for s in cutout_pickle.stencils] - assert expected == seen - - def test_serialize() -> None: cutout = CutoutParameters( ids=["foo"], stencils=[CircleStencil.from_string("1 1.42 1")], ).to_worker_parameters() - assert cutout.model_dump() == { + serialized = cutout.model_dump(mode="json") + assert serialized == { "dataset_ids": ["foo"], "stencils": [ { "type": "circle", - "center": (1.0, 1.42), + "center": [1.0, 1.42], "radius": 1.0, } ], } - assert cutout == WorkerCutout.model_validate(cutout.model_dump()) + assert cutout == WorkerCutout.model_validate(serialized) cutout = CutoutParameters( ids=["foo"], stencils=[PolygonStencil.from_string("1.2 0 1 1.4 0 1 0 0.5")], ).to_worker_parameters() - vertices = [(1.2, 0.0), (1.0, 1.4), (0.0, 1.0), (0.0, 0.5)] - assert cutout.model_dump() == { + vertices = [[1.2, 0.0], [1.0, 1.4], [0.0, 1.0], [0.0, 0.5]] + serialized = cutout.model_dump(mode="json") + assert serialized == { "dataset_ids": ["foo"], "stencils": [ { @@ -66,27 +49,28 @@ def test_serialize() -> None: # A SkyCoord with multiple coordinates cannot be compared with Python # equality, so we have to do this the hard way. - serialized_cutout = WorkerCutout.model_validate(cutout.model_dump()) - assert cutout.dataset_ids == serialized_cutout.dataset_ids - assert len(serialized_cutout.stencils) == 1 - assert isinstance(serialized_cutout.stencils[0], WorkerPolygonStencil) + unserialized_cutout = WorkerCutout.model_validate(serialized) + assert cutout.dataset_ids == unserialized_cutout.dataset_ids + assert len(unserialized_cutout.stencils) == 1 + assert isinstance(unserialized_cutout.stencils[0], WorkerPolygonStencil) assert vertices == [ - (float(v.ra.degree), float(v.dec.degree)) - for v in serialized_cutout.stencils[0].vertices + [float(v.ra.degree), float(v.dec.degree)] + for v in unserialized_cutout.stencils[0].vertices ] cutout = CutoutParameters( ids=["foo"], stencils=[RangeStencil.from_string("1 inf -inf 0")], ).to_worker_parameters() - assert cutout.model_dump() == { + serialized = cutout.model_dump(mode="json") + assert serialized == { "dataset_ids": ["foo"], "stencils": [ { "type": "range", - "ra": (1.0, math.inf), - "dec": (-math.inf, 0.0), + "ra": [1.0, math.inf], + "dec": [-math.inf, 0.0], } ], } - assert cutout == WorkerCutout.model_validate(cutout.model_dump()) + assert cutout == WorkerCutout.model_validate(serialized) diff --git a/tests/uws/job_api_test.py b/tests/uws/job_api_test.py index fcda26b..46c6ad2 100644 --- a/tests/uws/job_api_test.py +++ b/tests/uws/job_api_test.py @@ -22,7 +22,7 @@ from vocutouts.uws.models import UWSJob, UWSJobParameter, UWSJobResult from vocutouts.uws.uwsworker import WorkerJobInfo -from ..support.uws import MockJobRunner, SimpleWorkerParameters +from ..support.uws import MockJobRunner PENDING_JOB = """