From 2c400ac5c6499a1c518d7cb360851639aa5adbf6 Mon Sep 17 00:00:00 2001 From: stephen-riggs Date: Thu, 4 Jan 2024 15:47:35 +0000 Subject: [PATCH 1/9] Run pydantic migrator on murfey and fix issues that arise --- pyproject.toml | 3 +- src/murfey/client/instance_environment.py | 50 +++++++++++------------ src/murfey/client/tui/screens.py | 2 +- src/murfey/server/config.py | 7 ++-- src/murfey/server/demo_api.py | 3 +- src/murfey/server/main.py | 2 +- src/murfey/util/models.py | 32 +++++++-------- 7 files changed, 51 insertions(+), 48 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e0717d4b..72f4dd09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,8 @@ classifiers = [ ] dependencies = [ "defusedxml", # For safely parsing XML files - "pydantic<2", # Locked to <2 by zocalo + "pydantic>=2", + "pydantic_settings", "requests", "rich", "werkzeug", diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index 3c438750..b95d3d20 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -8,7 +8,8 @@ from typing import Callable, Dict, List, NamedTuple, Optional, Set from urllib.parse import ParseResult -from pydantic import BaseModel, validator +from pydantic import BaseModel, ConfigDict, field_validator +from pydantic_core.core_schema import FieldValidationInfo from murfey.client.watchdir import DirWatcher @@ -40,20 +41,20 @@ class MurfeyInstanceEnvironment(BaseModel): destination_registry: Dict[str, str] = {} watchers: Dict[Path, DirWatcher] = {} demo: bool = False - data_collection_group_ids: Dict[str, int] = {} - data_collection_ids: Dict[str, int] = {} - processing_job_ids: Dict[str, Dict[str, int]] = {} - autoproc_program_ids: Dict[str, Dict[str, int]] = {} id_tag_registry: Dict[str, List[str]] = { "data_collection_group": [], "data_collection": [], "processing_job": [], "auto_proc_program": [], } + listeners: Dict[str, Set[Callable]] = {} + data_collection_group_ids: Dict[str, int] = {} + data_collection_ids: Dict[str, int] = {} + processing_job_ids: Dict[str, Dict[str, int]] = {} + autoproc_program_ids: Dict[str, Dict[str, int]] = {} data_collection_parameters: dict = {} movies: Dict[Path, MovieTracker] = {} motion_corrected_movies: Dict[Path, List[str]] = {} - listeners: Dict[str, Set[Callable]] = {} movie_tilt_pair: Dict[Path, str] = {} tilt_angles: Dict[str, List[List[str]]] = {} movie_counters: Dict[str, itertools.count] = {} @@ -64,42 +65,41 @@ class MurfeyInstanceEnvironment(BaseModel): murfey_session: Optional[int] = None samples: Dict[Path, SampleInfo] = {} - class Config: - validate_assignment: bool = True - arbitrary_types_allowed: bool = True + model_config = ConfigDict(arbitrary_types_allowed=True) - @validator("data_collection_group_ids") - def dcg_callback(cls, v, values): + @field_validator("data_collection_group_ids") + def dcg_callback(cls, v, info: FieldValidationInfo): with global_env_lock: - for l in values.get("listeners", {}).get("data_collection_group_ids", []): + for l in info.data.get("listeners", {}).get( + "data_collection_group_ids", [] + ): for k in v.keys(): - if k not in values["id_tag_registry"]["data_collection"]: + if k not in info.data["id_tag_registry"]["data_collection"]: l(k) return v - @validator("data_collection_ids") - def dc_callback(cls, v, values): + @field_validator("data_collection_ids") + def dc_callback(cls, v, info: FieldValidationInfo): with global_env_lock: - for l in values.get("listeners", {}).get("data_collection_ids", []): + for l in info.data.get("listeners", {}).get("data_collection_ids", []): for k in v.keys(): - if k not in values["id_tag_registry"]["processing_job"]: + if k not in info.data["id_tag_registry"]["processing_job"]: l(k) return v - @validator("processing_job_ids") - def job_callback(cls, v, values): + @field_validator("processing_job_ids") + def job_callback(cls, v, info: FieldValidationInfo): with global_env_lock: - for l in values.get("listeners", {}).get("processing_job_ids", []): + for l in info.data.get("listeners", {}).get("processing_job_ids", []): for k in v.keys(): - if k not in values["id_tag_registry"]["auto_proc_program"]: + if k not in info.data["id_tag_registry"]["auto_proc_program"]: l(k, v[k]["ispyb-relion"]) return v - @validator("autoproc_program_ids") - def app_callback(cls, v, values): - # logger.info(f"autoproc program ids validator: {v}") + @field_validator("autoproc_program_ids") + def app_callback(cls, v, info: FieldValidationInfo): with global_env_lock: - for l in values.get("listeners", {}).get("autoproc_program_ids", []): + for l in info.data.get("listeners", {}).get("autoproc_program_ids", []): for k in v.keys(): if v[k].get("em-tomo-preprocess"): l(k, v[k]["em-tomo-preprocess"]) diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index a262841b..00b48f82 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -197,7 +197,7 @@ def validate_form(form: dict, model: BaseModel) -> bool: try: convert = lambda x: None if x == "None" else x validated = model(**{k: convert(v) for k, v in form.items()}) - log.info(validated.dict()) + log.info(validated.model_dump()) return True except (AttributeError, ValidationError) as e: log.warning(f"Form validation failed: {str(e)}") diff --git a/src/murfey/server/config.py b/src/murfey/server/config.py index c7c6958a..398c78c7 100644 --- a/src/murfey/server/config.py +++ b/src/murfey/server/config.py @@ -7,12 +7,13 @@ from typing import Dict, List, Optional, Union import yaml -from pydantic import BaseModel, BaseSettings +from pydantic import BaseModel +from pydantic_settings import BaseSettings class MachineConfig(BaseModel): acquisition_software: List[str] - calibrations: Dict[str, Dict[str, Union[dict, float]]] + calibrations: Dict[str, Dict[int, Union[dict, float]]] data_directories: Dict[Path, str] rsync_basepath: Path murfey_db_credentials: str @@ -20,7 +21,7 @@ class MachineConfig(BaseModel): default_model: Path display_name: str = "" image_path: Optional[Path] = None - software_versions: Dict[str, str] = {} + software_versions: Dict[str, float] = {} external_executables: Dict[str, str] = {} external_executables_eer: Dict[str, str] = {} external_environment: Dict[str, str] = {} diff --git a/src/murfey/server/demo_api.py b/src/murfey/server/demo_api.py index 4038d847..d0c326e8 100644 --- a/src/murfey/server/demo_api.py +++ b/src/murfey/server/demo_api.py @@ -14,7 +14,8 @@ from fastapi.responses import FileResponse, HTMLResponse from ispyb.sqlalchemy import BLSession from PIL import Image -from pydantic import BaseModel, BaseSettings +from pydantic import BaseModel +from pydantic_settings import BaseSettings from sqlalchemy import func from sqlmodel import col, select from werkzeug.utils import secure_filename diff --git a/src/murfey/server/main.py b/src/murfey/server/main.py index 0c19342f..a8c06c95 100644 --- a/src/murfey/server/main.py +++ b/src/murfey/server/main.py @@ -8,7 +8,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles from prometheus_client import make_asgi_app -from pydantic import BaseSettings +from pydantic_settings import BaseSettings import murfey.server import murfey.server.api.auth diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index ab53825f..46ba5a67 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -219,13 +219,13 @@ class SPAProcessFile(BaseModel): tag: str path: str description: str - processing_job: Optional[int] - data_collection_id: Optional[int] + processing_job: Optional[int] = None + data_collection_id: Optional[int] = None image_number: int - autoproc_program_id: Optional[int] - foil_hole_id: Optional[int] - pixel_size: Optional[float] - dose_per_frame: Optional[float] + autoproc_program_id: Optional[int] = None + foil_hole_id: Optional[int] = None + pixel_size: Optional[float] = None + dose_per_frame: Optional[float] = None mc_binning: Optional[int] = 1 gain_ref: Optional[str] = None extract_downscale: bool = True @@ -236,7 +236,7 @@ class SPAProcessFile(BaseModel): class ProcessingParametersSPA(BaseModel): tag: str dose_per_frame: float - gain_ref: Optional[str] + gain_ref: Optional[str] = None experiment_type: str voltage: float image_size_x: int @@ -247,12 +247,12 @@ class ProcessingParametersSPA(BaseModel): acquisition_software: str use_cryolo: bool symmetry: str - mask_diameter: Optional[int] - boxsize: Optional[int] + mask_diameter: Optional[int] = None + boxsize: Optional[int] = None downscale: bool - small_boxsize: Optional[int] + small_boxsize: Optional[int] = None eer_fractionation: int - particle_diameter: Optional[float] + particle_diameter: Optional[float] = None magnification: Optional[int] = None total_exposed_dose: Optional[float] = None c2aperture: Optional[float] = None @@ -261,14 +261,14 @@ class ProcessingParametersSPA(BaseModel): phase_plate: bool = False class Base(BaseModel): - dose_per_frame: Optional[float] - gain_ref: Optional[str] + dose_per_frame: Optional[float] = None + gain_ref: Optional[str] = None use_cryolo: bool symmetry: str - mask_diameter: Optional[int] - boxsize: Optional[int] + mask_diameter: Optional[int] = None + boxsize: Optional[int] = None downscale: bool - small_boxsize: Optional[int] + small_boxsize: Optional[int] = None eer_fractionation: int From ab06896e2f42a439549d04df469b0f29ed0f029e Mon Sep 17 00:00:00 2001 From: stephen-riggs Date: Thu, 4 Jan 2024 16:15:13 +0000 Subject: [PATCH 2/9] No more procrunner --- pyproject.toml | 1 - src/murfey/cli/transfer.py | 4 ++-- src/murfey/client/rsync.py | 33 ++++++++++++++++---------------- src/murfey/client/tui/app.py | 4 ++-- src/murfey/client/tui/screens.py | 4 ++-- src/murfey/util/rsync.py | 23 ++++++++++------------ 6 files changed, 32 insertions(+), 37 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 72f4dd09..ac79f567 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,6 @@ cicd = [ "pytest-cov", # Used by Azure Pipelines for PyTest coverage reports ] client = [ - "procrunner", "textual==0.42.0", "websocket-client", "xmltodict", diff --git a/src/murfey/cli/transfer.py b/src/murfey/cli/transfer.py index 01c1502d..a858f732 100644 --- a/src/murfey/cli/transfer.py +++ b/src/murfey/cli/transfer.py @@ -1,10 +1,10 @@ from __future__ import annotations import argparse +import subprocess from pathlib import Path from urllib.parse import urlparse -import procrunner import requests from rich.console import Console from rich.prompt import Confirm @@ -78,6 +78,6 @@ def run(): cmd.extend(list(Path(args.source or ".").glob("*"))) cmd.append(f"{murfey_url.hostname}::{args.destination}") - result = procrunner.run(cmd) + result = subprocess.run(cmd) if result.returncode: console.print(f"[red]rsync failed returning code {result.returncode}") diff --git a/src/murfey/client/rsync.py b/src/murfey/client/rsync.py index 2018e90a..f93c134a 100644 --- a/src/murfey/client/rsync.py +++ b/src/murfey/client/rsync.py @@ -18,8 +18,6 @@ from typing import List, NamedTuple from urllib.parse import ParseResult -import procrunner - from murfey.client.tui.status_bar import StatusBar from murfey.util import Observer @@ -408,28 +406,29 @@ def parse_stderr(line: str): result: subprocess.CompletedProcess | None = None success = True if rsync_stdin: - result = procrunner.run( + result = subprocess.run( rsync_cmd, - callback_stdout=parse_stdout, - callback_stderr=parse_stderr, - working_directory=str(self._basepath), - stdin=rsync_stdin, - print_stdout=False, - print_stderr=False, + cwd=str(self._basepath), + capture_output=True, + input=rsync_stdin, ) - success = result.returncode == 0 if result else False + for stdout_line in result.stdout.decode("utf8", "replace").split("\n"): + parse_stdout(stdout_line) + for stderr_line in result.stderr.decode("utf8", "replace").split("\n"): + parse_stderr(stderr_line) + success = result.returncode == 0 if rsync_stdin_remove: rsync_cmd.insert(-2, "--remove-source-files") - result = procrunner.run( + result = subprocess.run( rsync_cmd, - callback_stdout=parse_stdout, - callback_stderr=parse_stderr, - working_directory=str(self._basepath), - stdin=rsync_stdin_remove, - print_stdout=False, - print_stderr=False, + cwd=str(self._basepath), + input=rsync_stdin_remove, ) + for stdout_line in result.stdout.decode("utf8", "replace").split("\n"): + parse_stdout(stdout_line) + for stderr_line in result.stderr.decode("utf8", "replace").split("\n"): + parse_stderr(stderr_line) if success: success = result.returncode == 0 if result else False diff --git a/src/murfey/client/tui/app.py b/src/murfey/client/tui/app.py index 336633d9..2160da6c 100644 --- a/src/murfey/client/tui/app.py +++ b/src/murfey/client/tui/app.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import subprocess from datetime import datetime from functools import partial from pathlib import Path @@ -8,7 +9,6 @@ from typing import Awaitable, Callable, Dict, List, OrderedDict, TypeVar from urllib.parse import urlparse -import procrunner import requests from textual.app import App from textual.reactive import reactive @@ -203,7 +203,7 @@ def _start_rsyncer( if self._environment: self._environment.default_destinations[source] = destination if self._environment.gain_ref and visit_path: - gain_rsync = procrunner.run( + gain_rsync = subprocess.run( [ "rsync", str(self._environment.gain_ref), diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index 00b48f82..b96074ef 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -2,6 +2,7 @@ # import contextlib import logging +import subprocess from datetime import datetime from functools import partial from pathlib import Path @@ -17,7 +18,6 @@ TypeVar, ) -import procrunner import requests from pydantic import BaseModel, ValidationError from rich.box import SQUARE @@ -844,7 +844,7 @@ def on_button_pressed(self, event): if self.app._environment.demo: log.info(f"Would perform {' '.join(cmd)}") else: - gain_rsync = procrunner.run(cmd) + gain_rsync = subprocess.run(cmd) if gain_rsync.returncode: log.warning( f"Gain reference file {self._dir_tree._gain_reference} was not successfully transferred to {visit_path}/processing" diff --git a/src/murfey/util/rsync.py b/src/murfey/util/rsync.py index 2df38914..211ea00b 100644 --- a/src/murfey/util/rsync.py +++ b/src/murfey/util/rsync.py @@ -1,11 +1,10 @@ from __future__ import annotations import logging +import subprocess from pathlib import Path from typing import Callable, Dict, List, Optional, Tuple, Union -import procrunner - from murfey.util import Processor from murfey.util.file_monitor import Monitor @@ -32,7 +31,7 @@ def __init__( self.received_bytes = 0 self.byte_rate: float = 0 self.total_size = 0 - self.runner_return: List[procrunner.ReturnObject] = [] + self.runner_return: List[subprocess.CompletedProcess] = [] self._root = root self._sub_structure: Optional[Path] = None self._notify = notify or (lambda f: None) @@ -53,7 +52,7 @@ def _run_rsync( retry: bool = True, ): """ - Run rsync -v on a list of files using procrunner. + Run rsync -v on a list of files using subprocess run. :param root: root path of files for transferring; structure below the root is preserved :type root: pathlib.Path object @@ -109,11 +108,11 @@ def _single_rsync( else: cmd.append(str(self._finaldir / sub_struct) + "/") self._transferring = True - runner = procrunner.run( - cmd, - callback_stdout=self._parse_rsync_stdout, - callback_stderr=self._parse_rsync_stderr, - ) + + runner = subprocess.run(cmd, capture_output=True) + self._parse_rsync_stdout(runner.stdout) + self._parse_rsync_stderr(runner.stderr) + self.runner_return.append(runner) self.failed.extend(root / sub_struct / f for f in self._failed_tmp) if retry: @@ -127,8 +126,7 @@ def _parse_rsync_stdout(self, stdout: bytes): :param stdout: stdout of rsync process :type stdout: bytes """ - stringy_stdout = str(stdout) - if stringy_stdout: + for stringy_stdout in stdout.decode("utf8", "replace").split("\n"): if self._transferring: if stringy_stdout.startswith("sent"): self._transferring = False @@ -165,8 +163,7 @@ def _parse_rsync_stderr(self, stderr: bytes): :param stderr: stderr of rsync process :type stderr: bytes """ - stringy_stderr = str(stderr) - if stringy_stderr: + for stringy_stderr in stderr.decode("utf8", "replace").split("\n"): if ( stringy_stderr.startswith("rsync: link_stat") and "failed" in stringy_stderr From 32495bcd778aca3135353f6152ddb6d3a2cf5435 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 30 Aug 2024 13:37:50 +0100 Subject: [PATCH 3/9] Optionals got lost in rebase --- src/murfey/util/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 46ba5a67..470f98c2 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -94,7 +94,7 @@ class ConnectionFileParameters(BaseModel): class SessionInfo(BaseModel): - session_id: Optional[int] + session_id: Optional[int] = None session_name: str = "" rescale: bool = True @@ -181,7 +181,7 @@ class Sample(BaseModel): sample_group_id: int sample_id: int subsample_id: int - image_path: Optional[Path] + image_path: Optional[Path] = None class BLSampleImageParameters(BaseModel): @@ -312,7 +312,7 @@ class ProcessFile(BaseModel): # Rename to TomoProcessFile path: str description: str tag: str - data_collection_id: Optional[int] + data_collection_id: Optional[int] = None image_number: int pixel_size: float dose_per_frame: float @@ -352,7 +352,7 @@ class CompletedTiltSeries(BaseModel): class PreprocessingParametersTomo(BaseModel): dose_per_frame: float - gain_ref: Optional[str] + gain_ref: Optional[str] = None experiment_type: str voltage: float image_size_x: int @@ -363,12 +363,12 @@ class PreprocessingParametersTomo(BaseModel): file_extension: str tag: str tilt_series_tag: str - eer_fractionation_file: Optional[str] + eer_fractionation_file: Optional[str] = None eer_fractionation: int class Base(BaseModel): dose_per_frame: float - gain_ref: Optional[str] + gain_ref: Optional[str] = None manual_tilt_offset: float eer_fractionation: int From bd28c7b822915a717670194395d9561ad38a88ac Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 30 Aug 2024 13:41:17 +0100 Subject: [PATCH 4/9] This change seems wrong --- src/murfey/server/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/murfey/server/config.py b/src/murfey/server/config.py index 398c78c7..207adbdb 100644 --- a/src/murfey/server/config.py +++ b/src/murfey/server/config.py @@ -13,7 +13,7 @@ class MachineConfig(BaseModel): acquisition_software: List[str] - calibrations: Dict[str, Dict[int, Union[dict, float]]] + calibrations: Dict[str, Dict[str, Union[dict, float]]] data_directories: Dict[Path, str] rsync_basepath: Path murfey_db_credentials: str @@ -21,7 +21,7 @@ class MachineConfig(BaseModel): default_model: Path display_name: str = "" image_path: Optional[Path] = None - software_versions: Dict[str, float] = {} + software_versions: Dict[str, str] = {} external_executables: Dict[str, str] = {} external_executables_eer: Dict[str, str] = {} external_environment: Dict[str, str] = {} From 685b511099186d93d5a9024b3abe2c2b288c15b5 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 9 Oct 2024 09:26:15 +0100 Subject: [PATCH 5/9] Pydantic deprecation warning bits --- src/murfey/client/instance_environment.py | 10 +++++----- tests/cli/test_decrypt_password.py | 2 +- tests/cli/test_generate_password.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index 97bc1daf..1cf3dd01 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -9,7 +9,7 @@ from urllib.parse import ParseResult from pydantic import BaseModel, ConfigDict, field_validator -from pydantic_core.core_schema import FieldValidationInfo +from pydantic_core.core_schema import ValidationInfo from murfey.client.watchdir import DirWatcher @@ -69,7 +69,7 @@ class MurfeyInstanceEnvironment(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) @field_validator("data_collection_group_ids") - def dcg_callback(cls, v, info: FieldValidationInfo): + def dcg_callback(cls, v, info: ValidationInfo): with global_env_lock: for l in info.data.get("listeners", {}).get( "data_collection_group_ids", [] @@ -80,7 +80,7 @@ def dcg_callback(cls, v, info: FieldValidationInfo): return v @field_validator("data_collection_ids") - def dc_callback(cls, v, info: FieldValidationInfo): + def dc_callback(cls, v, info: ValidationInfo): with global_env_lock: for l in info.data.get("listeners", {}).get("data_collection_ids", []): for k in v.keys(): @@ -89,7 +89,7 @@ def dc_callback(cls, v, info: FieldValidationInfo): return v @field_validator("processing_job_ids") - def job_callback(cls, v, info: FieldValidationInfo): + def job_callback(cls, v, info: ValidationInfo): with global_env_lock: for l in info.data.get("listeners", {}).get("processing_job_ids", []): for k in v.keys(): @@ -98,7 +98,7 @@ def job_callback(cls, v, info: FieldValidationInfo): return v @field_validator("autoproc_program_ids") - def app_callback(cls, v, info: FieldValidationInfo): + def app_callback(cls, v, info: ValidationInfo): with global_env_lock: for l in info.data.get("listeners", {}).get("autoproc_program_ids", []): for k in v.keys(): diff --git a/tests/cli/test_decrypt_password.py b/tests/cli/test_decrypt_password.py index 65952e5e..f7b9a192 100644 --- a/tests/cli/test_decrypt_password.py +++ b/tests/cli/test_decrypt_password.py @@ -13,7 +13,7 @@ def test_decrypt_password(capsys, tmp_path): crypto_key = Fernet.generate_key() security_config.crypto_key = crypto_key.decode("ascii") with open(tmp_path / "config.yaml", "w") as cfg: - yaml.dump(security_config.dict(), cfg) + yaml.dump(security_config.model_dump(), cfg) os.environ["MURFEY_SECURITY_CONFIGURATION"] = str(tmp_path / "config.yaml") password = "abcd" f = Fernet(crypto_key) diff --git a/tests/cli/test_generate_password.py b/tests/cli/test_generate_password.py index fa48e9cf..e7d4ead0 100644 --- a/tests/cli/test_generate_password.py +++ b/tests/cli/test_generate_password.py @@ -12,7 +12,7 @@ def test_generate_password(capsys, tmp_path): crypto_key = Fernet.generate_key() security_config.crypto_key = crypto_key.decode("ascii") with open(tmp_path / "config.yaml", "w") as cfg: - yaml.dump(security_config.dict(), cfg) + yaml.dump(security_config.model_dump(), cfg) os.environ["MURFEY_SECURITY_CONFIGURATION"] = str(tmp_path / "config.yaml") run() captured = capsys.readouterr() From 4eb3cb75f840a7d0727f4cb9e4a35c0c0ecc23e9 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 11 Oct 2024 16:19:29 +0100 Subject: [PATCH 6/9] sqlalchemy2 change to metadata --- src/murfey/util/db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index ec39d900..6ef6151b 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -757,7 +757,8 @@ def setup(url: str): def clear(url: str): engine = create_engine(url) - metadata = sqlalchemy.MetaData(engine) - metadata.reflect() + metadata = sqlalchemy.MetaData() + metadata.create_all(engine) + metadata.reflect(engine) metadata.drop_all(engine) From d940ee37a0ce2bec5464fefb3844aa00d0c9d63c Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 14 Oct 2024 13:39:41 +0100 Subject: [PATCH 7/9] More pydantic 2 model dumps --- src/murfey/instrument_server/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py index cae6c3d3..8e9df951 100644 --- a/src/murfey/instrument_server/api.py +++ b/src/murfey/instrument_server/api.py @@ -143,7 +143,7 @@ def start_multigrid_watcher( demo=True, do_transfer=True, processing_enabled=not watcher_spec.skip_existing_processing, - _machine_config=watcher_spec.configuration.dict(), + _machine_config=watcher_spec.configuration.model_dump(), token=tokens.get(session_id, "token"), data_collection_parameters=data_collection_parameters.get(label, {}), ) @@ -156,7 +156,7 @@ def start_multigrid_watcher( (watcher_spec.source / d).mkdir(exist_ok=True) watchers[session_id] = MultigridDirWatcher( watcher_spec.source, - watcher_spec.configuration.dict(), + watcher_spec.configuration.model_dump(), skip_existing_processing=watcher_spec.skip_existing_processing, ) watchers[session_id].subscribe(controllers[session_id]._start_rsyncer_multigrid) @@ -221,7 +221,7 @@ def register_processing_parameters( session_id: MurfeySessionID, proc_param_block: ProcessingParameterBlock ): data_collection_parameters[proc_param_block.label] = {} - for k, v in proc_param_block.params.dict().items(): + for k, v in proc_param_block.params.model_dump().items(): data_collection_parameters[proc_param_block.label][k] = v return {"success": True} From ef3344554b53e7340080fea04785eb07abc5a2bf Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 14 Oct 2024 14:21:25 +0100 Subject: [PATCH 8/9] Pydantic doesn't like things starting model_ --- src/murfey/server/api/spa.py | 6 ++++-- src/murfey/util/config.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/murfey/server/api/spa.py b/src/murfey/server/api/spa.py index bc759425..6aa1caa7 100644 --- a/src/murfey/server/api/spa.py +++ b/src/murfey/server/api/spa.py @@ -20,7 +20,7 @@ def _cryolo_model_path(visit: str, instrument_name: str) -> Path: machine_config = get_machine_config(instrument_name=instrument_name)[ instrument_name ] - if machine_config.model_search_directory: + if machine_config.picking_model_search_directory: visit_directory = ( machine_config.rsync_basepath / (machine_config.rsync_module or "data") @@ -28,7 +28,9 @@ def _cryolo_model_path(visit: str, instrument_name: str) -> Path: / visit ) possible_models = list( - (visit_directory / machine_config.model_search_directory).glob("*.h5") + (visit_directory / machine_config.picking_model_search_directory).glob( + "*.h5" + ) ) if possible_models: return sorted(possible_models, key=lambda x: x.stat().st_ctime)[-1] diff --git a/src/murfey/util/config.py b/src/murfey/util/config.py index 70aacf79..5fccfaca 100644 --- a/src/murfey/util/config.py +++ b/src/murfey/util/config.py @@ -58,7 +58,7 @@ class MachineConfig(BaseModel): upstream_data_download_directory: Optional[Path] = None # Set by microscope config upstream_data_tiff_locations: List[str] = ["processed"] # Location of CLEM TIFFs - model_search_directory: str = "processing" + picking_model_search_directory: str = "processing" initial_model_search_directory: str = "processing/initial_model" failure_queue: str = "" From 72533df1921386b56b6378f173dec65151b18844 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 15 Oct 2024 09:16:08 +0100 Subject: [PATCH 9/9] Require zocalo>=1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2b9f195e..45a88edd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ server = [ "sqlmodel", "stomp-py<=8.1.0", # 8.1.1 (released 2024-04-06) doesn't work with our project "uvicorn[standard]", - "zocalo", + "zocalo>=1", ] [project.urls] Bug-Tracker = "https://github.com/DiamondLightSource/python-murfey/issues"