From 16159dce23a8be8f56b47b1cfcbc6e961946b3fb Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Tue, 30 Apr 2024 14:43:14 +0200 Subject: [PATCH 01/29] Move deployments and postprocessing into sailship function and make data directory a parameter instead of hardcoded --- virtual_ship/argo_deployments.py | 15 ++++----- virtual_ship/drifter_deployments.py | 13 ++++---- virtual_ship/sailship.py | 39 +++++++++++++++++----- virtual_ship/virtual_ship_configuration.py | 2 +- virtualship.py | 17 ++-------- 5 files changed, 47 insertions(+), 39 deletions(-) diff --git a/virtual_ship/argo_deployments.py b/virtual_ship/argo_deployments.py index 5a31870f..7c4600f3 100644 --- a/virtual_ship/argo_deployments.py +++ b/virtual_ship/argo_deployments.py @@ -16,7 +16,7 @@ ) -def argo_deployments(config, argo_time): +def argo_deployments(config, argo_time, data_dir: str): """ Deploy argo floats. @@ -28,7 +28,7 @@ def argo_deployments(config, argo_time): if len(config.argo_deploylocations) > 0: - fieldset = create_argo_fieldset(config) + fieldset = create_argo_fieldset(config, data_dir) # Define the new Kernel that mimics Argo vertical movement def ArgoVerticalMovement(particle, fieldset, time): @@ -148,19 +148,18 @@ class ArgoParticle(JITParticle): ) -def create_argo_fieldset(config): +def create_argo_fieldset(config, data_dir: str): """ Create a fieldset from netcdf files for argo floats, return fieldset with negative depth values. :param config: The cruise configuration. :returns: The fieldset. """ - datadirname = os.path.dirname(__file__) filenames = { - "U": os.path.join(datadirname, "argodata_UV.nc"), - "V": os.path.join(datadirname, "argodata_UV.nc"), - "S": os.path.join(datadirname, "argodata_S.nc"), - "T": os.path.join(datadirname, "argodata_T.nc"), + "U": os.path.join(data_dir, "argodata_UV.nc"), + "V": os.path.join(data_dir, "argodata_UV.nc"), + "S": os.path.join(data_dir, "argodata_S.nc"), + "T": os.path.join(data_dir, "argodata_T.nc"), } variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} dimensions = { diff --git a/virtual_ship/drifter_deployments.py b/virtual_ship/drifter_deployments.py index 89e19039..6bd95c15 100644 --- a/virtual_ship/drifter_deployments.py +++ b/virtual_ship/drifter_deployments.py @@ -8,7 +8,7 @@ from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable -def drifter_deployments(config, drifter_time): +def drifter_deployments(config, drifter_time, data_dir: str): """ Deploy drifters. @@ -17,7 +17,7 @@ def drifter_deployments(config, drifter_time): """ if len(config.drifter_deploylocations) > 0: - fieldset = create_drifter_fieldset(config) + fieldset = create_drifter_fieldset(config, data_dir) # Create particle to sample water underway class DrifterParticle(JITParticle): @@ -74,18 +74,17 @@ def CheckError(particle, fieldset, time): ) -def create_drifter_fieldset(config): +def create_drifter_fieldset(config, data_dir: str): """ Create a fieldset from netcdf files for drifters, returns fieldset with negative depth values. :param config: The cruise configuration. :returns: The fieldset. """ - datadirname = os.path.dirname(__file__) filenames = { - "U": os.path.join(datadirname, "drifterdata_UV.nc"), - "V": os.path.join(datadirname, "drifterdata_UV.nc"), - "T": os.path.join(datadirname, "drifterdata_T.nc"), + "U": os.path.join(data_dir, "drifterdata_UV.nc"), + "V": os.path.join(data_dir, "drifterdata_UV.nc"), + "T": os.path.join(data_dir, "drifterdata_T.nc"), } variables = {"U": "uo", "V": "vo", "T": "thetao"} dimensions = { diff --git a/virtual_ship/sailship.py b/virtual_ship/sailship.py index f753677d..ca6944c5 100644 --- a/virtual_ship/sailship.py +++ b/virtual_ship/sailship.py @@ -7,6 +7,10 @@ import pyproj from parcels import Field, FieldSet, JITParticle, ParticleSet, Variable from shapely.geometry import Point, Polygon +from .drifter_deployments import drifter_deployments +from .argo_deployments import argo_deployments +from .postprocess import postprocess +from .costs import costs def sailship(config): @@ -16,8 +20,12 @@ def sailship(config): :param config: The cruise configuration. :returns: drifter_time, argo_time, total_time """ + # hardcoding this until we refactor more + DATA_DIR = "data" + data_dir = DATA_DIR + # Create fieldset and retreive final schip route as sample_lons and sample_lats - fieldset = create_fieldset(config) + fieldset = create_fieldset(config, data_dir) sample_lons, sample_lats = shiproute(config) print("Arrived in region of interest, starting to gather data.") @@ -243,10 +251,24 @@ def SampleT(particle, fieldset, time): ) print("Cruise has ended. Please wait for drifters and/or Argo floats to finish.") - return drifter_time, argo_time, total_time + # simulate drifter deployments + drifter_deployments(config, drifter_time, data_dir) + + # simulate argo deployments + argo_deployments(config, argo_time, data_dir) + + # convert CTD data to CSV + postprocess() + + print("All data has been gathered and postprocessed, returning home.") + + cost = costs(config, total_time) + print( + f"This cruise took {timedelta(seconds=total_time)} and would have cost {cost:,.0f} euros." + ) -def create_fieldset(config): +def create_fieldset(config, data_dir: str): """ Create fieldset from netcdf files and adds bathymetry data for CTD cast, returns fieldset with negative depth values. @@ -254,12 +276,11 @@ def create_fieldset(config): :returns: The fieldset. :raises ValueError: If downloaded data is not as expected. """ - datadirname = os.path.dirname(__file__) filenames = { - "U": os.path.join(datadirname, "studentdata_UV.nc"), - "V": os.path.join(datadirname, "studentdata_UV.nc"), - "S": os.path.join(datadirname, "studentdata_S.nc"), - "T": os.path.join(datadirname, "studentdata_T.nc"), + "U": os.path.join(data_dir, "studentdata_UV.nc"), + "V": os.path.join(data_dir, "studentdata_UV.nc"), + "S": os.path.join(data_dir, "studentdata_S.nc"), + "T": os.path.join(data_dir, "studentdata_T.nc"), } variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} dimensions = { @@ -286,7 +307,7 @@ def create_fieldset(config): fieldset.add_constant("maxtime", fieldset.U.grid.time_full[-1]) # add bathymetry data to the fieldset for CTD cast - bathymetry_file = os.path.join(datadirname, "GLO-MFC_001_024_mask_bathy.nc") + bathymetry_file = os.path.join(data_dir, "GLO-MFC_001_024_mask_bathy.nc") bathymetry_variables = ("bathymetry", "deptho") bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} bathymetry_field = Field.from_netcdf( diff --git a/virtual_ship/virtual_ship_configuration.py b/virtual_ship/virtual_ship_configuration.py index fb796dc8..a872fd71 100644 --- a/virtual_ship/virtual_ship_configuration.py +++ b/virtual_ship/virtual_ship_configuration.py @@ -18,7 +18,7 @@ def __init__(self, json_file): :param json_file: Path to the JSON file to init from. :raises ValueError: If JSON file not valid. """ - with open(os.path.join(os.path.dirname(__file__), json_file), "r") as file: + with open(json_file, "r") as file: json_input = json.loads(file.read()) for key in json_input: setattr(self, key, json_input[key]) diff --git a/virtualship.py b/virtualship.py index 08774699..6ac71b70 100644 --- a/virtualship.py +++ b/virtualship.py @@ -1,17 +1,6 @@ -from datetime import timedelta from virtual_ship.virtual_ship_configuration import VirtualShipConfiguration -from virtual_ship.costs import costs from virtual_ship.sailship import sailship -from virtual_ship.drifter_deployments import drifter_deployments -from virtual_ship.argo_deployments import argo_deployments -from virtual_ship.postprocess import postprocess -if __name__ == '__main__': - config = VirtualShipConfiguration('student_input.json') - drifter_time, argo_time, total_time = sailship(config) - drifter_deployments(config, drifter_time) - argo_deployments(config, argo_time) - postprocess() - print("All data has been gathered and postprocessed, returning home.") - cost = costs(config, total_time) - print(f"This cruise took {timedelta(seconds=total_time)} and would have cost {cost:,.0f} euros.") +if __name__ == "__main__": + config = VirtualShipConfiguration("student_input.json") + sailship(config) From a41afab4014baab315b4fa7b93ef3c63f8922147 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Tue, 30 Apr 2024 17:49:39 +0200 Subject: [PATCH 02/29] ignore .vscode/ --- .gitignore | 174 ----------------------------------------------------- 1 file changed, 174 deletions(-) diff --git a/.gitignore b/.gitignore index 160e42a9..e69de29b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,174 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# Figures -*.jpg -*.gif - -# Data files created -/results -*.geojson -*.zarr -*.nc -*.zip - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# Auto generated by setuptools scm -virtual_ship/_version_setup.py From dbb7f0d7ed9d63a87e30160a08ba99844500943b Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Tue, 30 Apr 2024 17:52:04 +0200 Subject: [PATCH 03/29] ignore .vscode/ --- .gitignore | 176 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/.gitignore b/.gitignore index e69de29b..6f875b36 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,176 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Figures +*.jpg +*.gif + +# Data files created +/results +*.geojson +*.zarr +*.nc +*.zip + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Auto generated by setuptools scm +virtual_ship/_version_setup.py + +.vscode/ \ No newline at end of file From 34a44a7dd517b3105a8f966215f7d0971f2397a8 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Tue, 30 Apr 2024 18:09:02 +0200 Subject: [PATCH 04/29] Crudely move fieldset creation out of sailship and onto the user, passed through the config object. Create test that runs a very simple cruise --- pyproject.toml | 1 + tests/conftest.py | 7 +++ tests/sailship_config.json | 40 +++++++++++++++++ tests/test_sailship.py | 50 ++++++++++++++++++++++ virtual_ship/argo_deployments.py | 14 ++++-- virtual_ship/drifter_deployments.py | 5 ++- virtual_ship/sailship.py | 13 +++--- virtual_ship/virtual_ship_configuration.py | 17 +++++++- 8 files changed, 132 insertions(+), 15 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/sailship_config.json create mode 100644 tests/test_sailship.py diff --git a/pyproject.toml b/pyproject.toml index 69ce0bf2..9a012b3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ dev = [ "isort == 5.13.2", "pydocstyle == 6.3.0", "sort-all == 1.2.0", + "pytest == 8.2.0", ] [tool.isort] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..01e91653 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +import pytest + + +# Set the working directory for each test to the directory of that test. +@pytest.fixture(autouse=True) +def change_test_dir(request, monkeypatch): + monkeypatch.chdir(request.fspath.dirname) diff --git a/tests/sailship_config.json b/tests/sailship_config.json new file mode 100644 index 00000000..e406f7fc --- /dev/null +++ b/tests/sailship_config.json @@ -0,0 +1,40 @@ +{ + "region_of_interest": { + "North": 64, + "East": -23, + "South": 59, + "West": -43 + }, + "requested_ship_time": { + "start": "2022-01-01T00:00:00", + "end": "2022-01-2T00:00:00" + }, + "route_coordinates": [ + [-23.071289, 63.743631], [-23.081289, 63.743631], [-23.091289, 63.743631] + ], + "underway_data": true, + "ADCP_data": false, + "ADCP_settings": { + "max_depth": -1000, + "bin_size_m": 24 + }, + "CTD_locations": [ + [-23.071289, 63.743631] + ], + "CTD_settings": { + "max_depth": "max" + }, + "drifter_deploylocations": [ + + ], + "argo_deploylocations": [ + [-23.081289, 63.743631] + ], + "argo_characteristics": { + "driftdepth": -1000, + "maxdepth": -2000, + "vertical_speed": -0.10, + "cycle_days" : 10, + "drift_days": 9 + } + } \ No newline at end of file diff --git a/tests/test_sailship.py b/tests/test_sailship.py new file mode 100644 index 00000000..374f523f --- /dev/null +++ b/tests/test_sailship.py @@ -0,0 +1,50 @@ +"""Performs a complete cruise with virtual ship.""" + +from virtual_ship.virtual_ship_configuration import VirtualShipConfiguration +from virtual_ship.sailship import sailship +from parcels import FieldSet +import numpy as np + + +def test_sailship() -> None: + ctd_fieldset = FieldSet.from_data( + {"U": 0, "V": 0, "S": 0, "T": 0, "bathymetry": 0}, + {"lon": 0, "lat": 0}, + ) + ctd_fieldset.add_constant("maxtime", ctd_fieldset.U.grid.time_full[-1]) + ctd_fieldset.add_constant("mindepth", -ctd_fieldset.U.depth[0]) + ctd_fieldset.add_constant("max_depth", -ctd_fieldset.U.depth[-1]) + + drifter_fieldset = FieldSet.from_data( + { + "U": 0, + "V": 0, + }, + {"lon": 0, "lat": 0}, + ) + + argo_fieldset = FieldSet.from_data( + {"U": 0, "V": 0, "T": 0, "S": 0}, + { + "lon": 0, + "lat": 0, + "time": [np.datetime64("1950-01-01") + np.timedelta64(632160, "h")], + }, + ) + + config = VirtualShipConfiguration( + "sailship_config.json", + ctd_fieldset=ctd_fieldset, + drifter_fieldset=drifter_fieldset, + argo_fieldset=argo_fieldset, + ) + + argo_fieldset.add_constant("mindepth", -argo_fieldset.U.depth[0]) + argo_fieldset.add_constant("driftdepth", config.argo_characteristics["driftdepth"]) + argo_fieldset.add_constant("maxdepth", config.argo_characteristics["maxdepth"]) + argo_fieldset.add_constant( + "vertical_speed", config.argo_characteristics["vertical_speed"] + ) + argo_fieldset.add_constant("cycle_days", config.argo_characteristics["cycle_days"]) + argo_fieldset.add_constant("drift_days", config.argo_characteristics["drift_days"]) + sailship(config) diff --git a/virtual_ship/argo_deployments.py b/virtual_ship/argo_deployments.py index 7c4600f3..9a8507b3 100644 --- a/virtual_ship/argo_deployments.py +++ b/virtual_ship/argo_deployments.py @@ -15,8 +15,10 @@ Variable, ) +from .virtual_ship_configuration import VirtualShipConfiguration -def argo_deployments(config, argo_time, data_dir: str): + +def argo_deployments(config: VirtualShipConfiguration, argo_time): """ Deploy argo floats. @@ -28,7 +30,10 @@ def argo_deployments(config, argo_time, data_dir: str): if len(config.argo_deploylocations) > 0: - fieldset = create_argo_fieldset(config, data_dir) + # fieldset = create_argo_fieldset( + # config, "/home/astuurman/projects/Virtual_ship_classroom/data" + # ) + fieldset = config.argo_fieldset # Define the new Kernel that mimics Argo vertical movement def ArgoVerticalMovement(particle, fieldset, time): @@ -132,8 +137,9 @@ class ArgoParticle(JITParticle): config.requested_ship_time["start"], "%Y-%m-%dT%H:%M:%S" ) + timedelta(weeks=6) - ) - ).astype("datetime64[ms]") + ), + dtype="datetime64[ms]", + ) argoset.execute( [ diff --git a/virtual_ship/drifter_deployments.py b/virtual_ship/drifter_deployments.py index 6bd95c15..84961906 100644 --- a/virtual_ship/drifter_deployments.py +++ b/virtual_ship/drifter_deployments.py @@ -6,9 +6,10 @@ import numpy as np from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable +from .virtual_ship_configuration import VirtualShipConfiguration -def drifter_deployments(config, drifter_time, data_dir: str): +def drifter_deployments(config: VirtualShipConfiguration, drifter_time): """ Deploy drifters. @@ -17,7 +18,7 @@ def drifter_deployments(config, drifter_time, data_dir: str): """ if len(config.drifter_deploylocations) > 0: - fieldset = create_drifter_fieldset(config, data_dir) + fieldset = config.drifter_fieldset # Create particle to sample water underway class DrifterParticle(JITParticle): diff --git a/virtual_ship/sailship.py b/virtual_ship/sailship.py index ca6944c5..1e46f49e 100644 --- a/virtual_ship/sailship.py +++ b/virtual_ship/sailship.py @@ -11,21 +11,18 @@ from .argo_deployments import argo_deployments from .postprocess import postprocess from .costs import costs +from .virtual_ship_configuration import VirtualShipConfiguration -def sailship(config): +def sailship(config: VirtualShipConfiguration): """ Use parcels to simulate the ship, take CTDs and measure ADCP and underwaydata. :param config: The cruise configuration. :returns: drifter_time, argo_time, total_time """ - # hardcoding this until we refactor more - DATA_DIR = "data" - data_dir = DATA_DIR - # Create fieldset and retreive final schip route as sample_lons and sample_lats - fieldset = create_fieldset(config, data_dir) + fieldset = config.ctd_fieldset # create_fieldset(config, data_dir) sample_lons, sample_lats = shiproute(config) print("Arrived in region of interest, starting to gather data.") @@ -252,10 +249,10 @@ def SampleT(particle, fieldset, time): print("Cruise has ended. Please wait for drifters and/or Argo floats to finish.") # simulate drifter deployments - drifter_deployments(config, drifter_time, data_dir) + drifter_deployments(config, drifter_time) # simulate argo deployments - argo_deployments(config, argo_time, data_dir) + argo_deployments(config, argo_time) # convert CTD data to CSV postprocess() diff --git a/virtual_ship/virtual_ship_configuration.py b/virtual_ship/virtual_ship_configuration.py index a872fd71..841691ab 100644 --- a/virtual_ship/virtual_ship_configuration.py +++ b/virtual_ship/virtual_ship_configuration.py @@ -6,18 +6,33 @@ from datetime import timedelta from shapely.geometry import Point, Polygon +from parcels import FieldSet class VirtualShipConfiguration: """Configuration of the virtual ship, initialized from a json file.""" - def __init__(self, json_file): + ctd_fieldset: FieldSet + drifter_fieldset: FieldSet + argo_fieldset: FieldSet + + def __init__( + self, + json_file, + ctd_fieldset: FieldSet, + drifter_fieldset: FieldSet, + argo_fieldset: FieldSet, + ): """ Initialize this object. :param json_file: Path to the JSON file to init from. :raises ValueError: If JSON file not valid. """ + self.ctd_fieldset = ctd_fieldset + self.drifter_fieldset = drifter_fieldset + self.argo_fieldset = argo_fieldset + with open(json_file, "r") as file: json_input = json.loads(file.read()) for key in json_input: From 56f44b561b288f3dfaaf919fba88403f4a9079fb Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Wed, 22 May 2024 12:44:17 +0200 Subject: [PATCH 05/29] Create sensors module, starting with argos. create basic test for argos. --- tests/sensors/test_argo.py | 36 +++++++ virtual_ship/sensors/argo.py | 169 +++++++++++++++++++++++++++++++ virtual_ship/sensors/location.py | 15 +++ 3 files changed, 220 insertions(+) create mode 100644 tests/sensors/test_argo.py create mode 100644 virtual_ship/sensors/argo.py create mode 100644 virtual_ship/sensors/location.py diff --git a/tests/sensors/test_argo.py b/tests/sensors/test_argo.py new file mode 100644 index 00000000..b55082ff --- /dev/null +++ b/tests/sensors/test_argo.py @@ -0,0 +1,36 @@ +"""Test the simulation of argos.""" + +from virtual_ship.sensors.argo import simulate_argos, Argo +from virtual_ship.sensors.location import Location +from parcels import FieldSet +import numpy as np + + +def test_simulate_argos() -> None: + DRIFT_DEPTH = -1000 + MAX_DEPTH = -2000 + VERTICLE_SPEED = -0.10 + CYCLE_DAYS = 10 + DRIFT_DAYS = 9 + + environment = FieldSet.from_data( + {"U": 0, "V": 0, "T": 0, "S": 0}, + { + "lon": 0, + "lat": 0, + "time": [np.datetime64("1950-01-01") + np.timedelta64(632160, "h")], + }, + ) + + argos = [Argo(location=Location(latitude=0, longitude=0), deployment_time=0)] + + simulate_argos( + argos=argos, + environment=environment, + out_file_name="test", + drift_depth=DRIFT_DEPTH, + max_depth=MAX_DEPTH, + verticle_speed=VERTICLE_SPEED, + cycle_days=CYCLE_DAYS, + drift_days=DRIFT_DAYS, + ) diff --git a/virtual_ship/sensors/argo.py b/virtual_ship/sensors/argo.py new file mode 100644 index 00000000..7eae959f --- /dev/null +++ b/virtual_ship/sensors/argo.py @@ -0,0 +1,169 @@ +from .location import Location +from dataclasses import dataclass +from parcels import ( + ParticleSet, + JITParticle, + Variable, + FieldSet, + AdvectionRK4, + StatusCode, +) +import math +import numpy as np +from datetime import timedelta + + +@dataclass +class Argo: + location: Location + deployment_time: float + + +class _ArgoParticle(JITParticle): + cycle_phase = Variable("cycle_phase", dtype=np.int32, initial=0.0) + cycle_age = Variable("cycle_age", dtype=np.float32, initial=0.0) + drift_age = Variable("drift_age", dtype=np.float32, initial=0.0) + salinity = Variable("salinity", initial=np.nan) + temperature = Variable("temperature", initial=np.nan) + + +def _argo_vertical_movement(particle, fieldset, time): + + if particle.cycle_phase == 0: + # Phase 0: Sinking with vertical_speed until depth is driftdepth + particle_ddepth += ( # noqa See comment above about particle_* variables. + fieldset.vertical_speed * particle.dt + ) + if particle.depth + particle_ddepth <= fieldset.driftdepth: + particle_ddepth = fieldset.driftdepth - particle.depth + particle.cycle_phase = 1 + + elif particle.cycle_phase == 1: + # Phase 1: Drifting at depth for drifttime seconds + particle.drift_age += particle.dt + if particle.drift_age >= fieldset.drift_days * 86400: + particle.drift_age = 0 # reset drift_age for next cycle + particle.cycle_phase = 2 + + elif particle.cycle_phase == 2: + # Phase 2: Sinking further to maxdepth + particle_ddepth += fieldset.vertical_speed * particle.dt + if particle.depth + particle_ddepth <= fieldset.maxdepth: + particle_ddepth = fieldset.maxdepth - particle.depth + particle.cycle_phase = 3 + + elif particle.cycle_phase == 3: + # Phase 3: Rising with vertical_speed until at surface + particle_ddepth -= fieldset.vertical_speed * particle.dt + particle.cycle_age += ( + particle.dt + ) # solve issue of not updating cycle_age during ascent + if particle.depth + particle_ddepth >= fieldset.min_depth: + particle_ddepth = fieldset.min_depth - particle.depth + particle.temperature = ( + math.nan + ) # reset temperature to NaN at end of sampling cycle + particle.salinity = math.nan # idem + particle.cycle_phase = 4 + else: + particle.temperature = fieldset.T[ + time, particle.depth, particle.lat, particle.lon + ] + particle.salinity = fieldset.S[ + time, particle.depth, particle.lat, particle.lon + ] + + elif particle.cycle_phase == 4: + # Phase 4: Transmitting at surface until cycletime is reached + if particle.cycle_age > fieldset.cycle_days * 86400: + particle.cycle_phase = 0 + particle.cycle_age = 0 + + if particle.state == StatusCode.Evaluate: + particle.cycle_age += particle.dt # update cycle_age + + +def _keep_at_surface(particle, fieldset, time): + # Prevent error when float reaches surface + if particle.state == StatusCode.ErrorThroughSurface: + particle.depth = fieldset.min_depth + particle.state = StatusCode.Success + + +def _check_error(particle, fieldset, time): + if particle.state >= 50: # This captures all Errors + particle.delete() + + +def simulate_argos( + argos: list[Argo], + environment: FieldSet, + out_file_name: str, + max_depth: float, + drift_depth: float, + verticle_speed: float, + cycle_days: float, + drift_days: float, +) -> None: + """ + Use parcels to simulate a set of argos in a fieldset. + + :param argos: The argos to simulate. + :param environment: The environment to simulate the argos in. + :param out_file_name: The file to write the results to. + :param max_depth: TODO + :param drift_depth: TODO + :param verticle_speed: TODO + :param cycle_days: TODO + :param drift_days: TODO + """ + + lon = [argo.location.lon for argo in argos] + lat = [argo.location.lat for argo in argos] + time = [argo.deployment_time for argo in argos] + + min_depth = -environment.U.depth[0] + + # define the parcels simulation + argoset = ParticleSet( + fieldset=environment, + pclass=_ArgoParticle, + lon=lon, + lat=lat, + depth=np.repeat(min_depth, len(argos)), + time=time, + ) + + # define output file for the simulation + out_file = argoset.ParticleFile( + name=out_file_name, + outputdt=timedelta(minutes=5), + chunks=(1, 500), + ) + + # get time when the fieldset ends + fieldset_endtime = environment.time_origin.fulltime( + environment.U.grid.time_full[-1] + ) + + # set constants on environment fieldset required by kernels. + # sadly we must change the fieldset parameter to pass this information + environment.add_constant("min_depth", min_depth) + environment.add_constant("maxdepth", max_depth) + environment.add_constant("driftdepth", drift_depth) + environment.add_constant("vertical_speed", verticle_speed) + environment.add_constant("cycle_days", cycle_days) + environment.add_constant("drift_days", drift_days) + + # execute simulation + argoset.execute( + [ + _argo_vertical_movement, + AdvectionRK4, + _keep_at_surface, + _check_error, + ], + endtime=fieldset_endtime, + dt=timedelta(minutes=5), + output_file=out_file, + ) diff --git a/virtual_ship/sensors/location.py b/virtual_ship/sensors/location.py new file mode 100644 index 00000000..5807485e --- /dev/null +++ b/virtual_ship/sensors/location.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Location: + latitude: float + longitude: float + + @property + def lat(self) -> float: + return self.latitude + + @property + def lon(self) -> float: + return self.longitude From fd6f84e77f90d8a377a3826023f8486ad1f889c7 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Wed, 22 May 2024 12:58:39 +0200 Subject: [PATCH 06/29] Use argo sensor in sailship function --- tests/test_sailship.py | 8 -------- virtual_ship/sailship.py | 28 ++++++++++++++++++++++++---- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/tests/test_sailship.py b/tests/test_sailship.py index 374f523f..845fe141 100644 --- a/tests/test_sailship.py +++ b/tests/test_sailship.py @@ -39,12 +39,4 @@ def test_sailship() -> None: argo_fieldset=argo_fieldset, ) - argo_fieldset.add_constant("mindepth", -argo_fieldset.U.depth[0]) - argo_fieldset.add_constant("driftdepth", config.argo_characteristics["driftdepth"]) - argo_fieldset.add_constant("maxdepth", config.argo_characteristics["maxdepth"]) - argo_fieldset.add_constant( - "vertical_speed", config.argo_characteristics["vertical_speed"] - ) - argo_fieldset.add_constant("cycle_days", config.argo_characteristics["cycle_days"]) - argo_fieldset.add_constant("drift_days", config.argo_characteristics["drift_days"]) sailship(config) diff --git a/virtual_ship/sailship.py b/virtual_ship/sailship.py index 1e46f49e..f5aa1341 100644 --- a/virtual_ship/sailship.py +++ b/virtual_ship/sailship.py @@ -12,6 +12,8 @@ from .postprocess import postprocess from .costs import costs from .virtual_ship_configuration import VirtualShipConfiguration +from .sensors.argo import Argo, simulate_argos +from .sensors.location import Location def sailship(config: VirtualShipConfiguration): @@ -137,7 +139,7 @@ def SampleT(particle, fieldset, time): drifter = 0 drifter_time = [] argo = 0 - argo_time = [] + argos: list[Argo] = [] # run the model for the length of the sample_lons list for i in range(len(sample_lons) - 1): @@ -160,7 +162,8 @@ def SampleT(particle, fieldset, time): print( "Ship time is over, waiting for drifters and/or Argo floats to finish." ) - return drifter_time, argo_time + raise NotImplementedError() + # return drifter_time, argo_time # check if virtual ship is at a CTD station if ctd < len(config.CTD_locations): @@ -219,7 +222,15 @@ def SampleT(particle, fieldset, time): abs(sample_lons[i] - config.argo_deploylocations[argo][0]) < 0.01 and abs(sample_lats[i] - config.argo_deploylocations[argo][1]) < 0.01 ): - argo_time.append(total_time) + argos.append( + Argo( + location=Location( + latitude=config.argo_deploylocations[argo][0], + longitude=config.argo_deploylocations[argo][1], + ), + deployment_time=total_time, + ) + ) argo += 1 print(f"Argo {argo} deployed at {sample_lons[i]}, {sample_lats[i]}") if argo == len(config.argo_deploylocations): @@ -252,7 +263,16 @@ def SampleT(particle, fieldset, time): drifter_deployments(config, drifter_time) # simulate argo deployments - argo_deployments(config, argo_time) + simulate_argos( + argos=argos, + environment=config.argo_fieldset, + out_file_name=os.path.join("results", "Argos.zarr"), + max_depth=config.argo_characteristics["maxdepth"], + drift_depth=config.argo_characteristics["driftdepth"], + verticle_speed=config.argo_characteristics["vertical_speed"], + cycle_days=config.argo_characteristics["cycle_days"], + drift_days=config.argo_characteristics["drift_days"], + ) # convert CTD data to CSV postprocess() From 17948e1cdd5cfd678cb0602f3ce7632c14a324f8 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 13:40:20 +0200 Subject: [PATCH 07/29] rename sensors to instruments --- virtual_ship/{sensors => instruments}/argo.py | 0 virtual_ship/{sensors => instruments}/location.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename virtual_ship/{sensors => instruments}/argo.py (100%) rename virtual_ship/{sensors => instruments}/location.py (100%) diff --git a/virtual_ship/sensors/argo.py b/virtual_ship/instruments/argo.py similarity index 100% rename from virtual_ship/sensors/argo.py rename to virtual_ship/instruments/argo.py diff --git a/virtual_ship/sensors/location.py b/virtual_ship/instruments/location.py similarity index 100% rename from virtual_ship/sensors/location.py rename to virtual_ship/instruments/location.py From 0a6ab7a7cf689bd684ebef5ea04e10bc9571f445 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 13:42:56 +0200 Subject: [PATCH 08/29] rename sensors to instruments --- tests/sensors/test_argo.py | 8 ++++---- virtual_ship/instruments/argo.py | 26 ++++++++++++-------------- virtual_ship/sailship.py | 6 +++--- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/tests/sensors/test_argo.py b/tests/sensors/test_argo.py index b55082ff..a9f63ca4 100644 --- a/tests/sensors/test_argo.py +++ b/tests/sensors/test_argo.py @@ -1,7 +1,7 @@ """Test the simulation of argos.""" -from virtual_ship.sensors.argo import simulate_argos, Argo -from virtual_ship.sensors.location import Location +from virtual_ship.instruments.argo import simulate_argos, Argo +from virtual_ship.instruments.location import Location from parcels import FieldSet import numpy as np @@ -13,7 +13,7 @@ def test_simulate_argos() -> None: CYCLE_DAYS = 10 DRIFT_DAYS = 9 - environment = FieldSet.from_data( + fieldset = FieldSet.from_data( {"U": 0, "V": 0, "T": 0, "S": 0}, { "lon": 0, @@ -26,7 +26,7 @@ def test_simulate_argos() -> None: simulate_argos( argos=argos, - environment=environment, + fieldset=fieldset, out_file_name="test", drift_depth=DRIFT_DEPTH, max_depth=MAX_DEPTH, diff --git a/virtual_ship/instruments/argo.py b/virtual_ship/instruments/argo.py index 7eae959f..06f061dd 100644 --- a/virtual_ship/instruments/argo.py +++ b/virtual_ship/instruments/argo.py @@ -97,7 +97,7 @@ def _check_error(particle, fieldset, time): def simulate_argos( argos: list[Argo], - environment: FieldSet, + fieldset: FieldSet, out_file_name: str, max_depth: float, drift_depth: float, @@ -109,7 +109,7 @@ def simulate_argos( Use parcels to simulate a set of argos in a fieldset. :param argos: The argos to simulate. - :param environment: The environment to simulate the argos in. + :param fieldset: The fieldset to simulate the argos in. :param out_file_name: The file to write the results to. :param max_depth: TODO :param drift_depth: TODO @@ -122,11 +122,11 @@ def simulate_argos( lat = [argo.location.lat for argo in argos] time = [argo.deployment_time for argo in argos] - min_depth = -environment.U.depth[0] + min_depth = -fieldset.U.depth[0] # define the parcels simulation argoset = ParticleSet( - fieldset=environment, + fieldset=fieldset, pclass=_ArgoParticle, lon=lon, lat=lat, @@ -142,18 +142,16 @@ def simulate_argos( ) # get time when the fieldset ends - fieldset_endtime = environment.time_origin.fulltime( - environment.U.grid.time_full[-1] - ) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - # set constants on environment fieldset required by kernels. + # set constants on fieldset required by kernels. # sadly we must change the fieldset parameter to pass this information - environment.add_constant("min_depth", min_depth) - environment.add_constant("maxdepth", max_depth) - environment.add_constant("driftdepth", drift_depth) - environment.add_constant("vertical_speed", verticle_speed) - environment.add_constant("cycle_days", cycle_days) - environment.add_constant("drift_days", drift_days) + fieldset.add_constant("min_depth", min_depth) + fieldset.add_constant("maxdepth", max_depth) + fieldset.add_constant("driftdepth", drift_depth) + fieldset.add_constant("vertical_speed", verticle_speed) + fieldset.add_constant("cycle_days", cycle_days) + fieldset.add_constant("drift_days", drift_days) # execute simulation argoset.execute( diff --git a/virtual_ship/sailship.py b/virtual_ship/sailship.py index f5aa1341..2536c62b 100644 --- a/virtual_ship/sailship.py +++ b/virtual_ship/sailship.py @@ -12,8 +12,8 @@ from .postprocess import postprocess from .costs import costs from .virtual_ship_configuration import VirtualShipConfiguration -from .sensors.argo import Argo, simulate_argos -from .sensors.location import Location +from .instruments.argo import Argo, simulate_argos +from .instruments.location import Location def sailship(config: VirtualShipConfiguration): @@ -265,7 +265,7 @@ def SampleT(particle, fieldset, time): # simulate argo deployments simulate_argos( argos=argos, - environment=config.argo_fieldset, + fieldset=config.argo_fieldset, out_file_name=os.path.join("results", "Argos.zarr"), max_depth=config.argo_characteristics["maxdepth"], drift_depth=config.argo_characteristics["driftdepth"], From c2bb72f8fefb5feb0a6c49f7f969414cc25f7a8c Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 13:50:23 +0200 Subject: [PATCH 09/29] Resolved codetools errors --- codetools.sh | 0 virtual_ship/argo_deployments.py | 194 --------------------- virtual_ship/drifter_deployments.py | 2 + virtual_ship/instruments/argo.py | 23 ++- virtual_ship/instruments/location.py | 14 ++ virtual_ship/sailship.py | 11 +- virtual_ship/virtual_ship_configuration.py | 6 +- 7 files changed, 40 insertions(+), 210 deletions(-) mode change 100644 => 100755 codetools.sh delete mode 100644 virtual_ship/argo_deployments.py diff --git a/codetools.sh b/codetools.sh old mode 100644 new mode 100755 diff --git a/virtual_ship/argo_deployments.py b/virtual_ship/argo_deployments.py deleted file mode 100644 index 9a8507b3..00000000 --- a/virtual_ship/argo_deployments.py +++ /dev/null @@ -1,194 +0,0 @@ -"""argo_deployments function.""" - -import datetime -import math -import os -from datetime import timedelta - -import numpy as np -from parcels import ( - AdvectionRK4, - FieldSet, - JITParticle, - ParticleSet, - StatusCode, - Variable, -) - -from .virtual_ship_configuration import VirtualShipConfiguration - - -def argo_deployments(config: VirtualShipConfiguration, argo_time): - """ - Deploy argo floats. - - :param config: The cruise configuration. - :param argo_time: TODO - """ - # particle_* such as particle_ddepth are local variables defined by parcels. - # See https://docs.oceanparcels.org/en/latest/examples/tutorial_kernelloop.html#Background - - if len(config.argo_deploylocations) > 0: - - # fieldset = create_argo_fieldset( - # config, "/home/astuurman/projects/Virtual_ship_classroom/data" - # ) - fieldset = config.argo_fieldset - - # Define the new Kernel that mimics Argo vertical movement - def ArgoVerticalMovement(particle, fieldset, time): - - if particle.cycle_phase == 0: - # Phase 0: Sinking with vertical_speed until depth is driftdepth - particle_ddepth += ( # noqa See comment above about particle_* variables. - fieldset.vertical_speed * particle.dt - ) - if particle.depth + particle_ddepth <= fieldset.driftdepth: - particle_ddepth = fieldset.driftdepth - particle.depth - particle.cycle_phase = 1 - - elif particle.cycle_phase == 1: - # Phase 1: Drifting at depth for drifttime seconds - particle.drift_age += particle.dt - if particle.drift_age >= fieldset.drift_days * 86400: - particle.drift_age = 0 # reset drift_age for next cycle - particle.cycle_phase = 2 - - elif particle.cycle_phase == 2: - # Phase 2: Sinking further to maxdepth - particle_ddepth += fieldset.vertical_speed * particle.dt - if particle.depth + particle_ddepth <= fieldset.maxdepth: - particle_ddepth = fieldset.maxdepth - particle.depth - particle.cycle_phase = 3 - - elif particle.cycle_phase == 3: - # Phase 3: Rising with vertical_speed until at surface - particle_ddepth -= fieldset.vertical_speed * particle.dt - particle.cycle_age += ( - particle.dt - ) # solve issue of not updating cycle_age during ascent - if particle.depth + particle_ddepth >= fieldset.mindepth: - particle_ddepth = fieldset.mindepth - particle.depth - particle.temperature = ( - math.nan - ) # reset temperature to NaN at end of sampling cycle - particle.salinity = math.nan # idem - particle.cycle_phase = 4 - else: - particle.temperature = fieldset.T[ - time, particle.depth, particle.lat, particle.lon - ] - particle.salinity = fieldset.S[ - time, particle.depth, particle.lat, particle.lon - ] - - elif particle.cycle_phase == 4: - # Phase 4: Transmitting at surface until cycletime is reached - if particle.cycle_age > fieldset.cycle_days * 86400: - particle.cycle_phase = 0 - particle.cycle_age = 0 - - if particle.state == StatusCode.Evaluate: - particle.cycle_age += particle.dt # update cycle_age - - def KeepAtSurface(particle, fieldset, time): - # Prevent error when float reaches surface - if particle.state == StatusCode.ErrorThroughSurface: - particle.depth = fieldset.mindepth - particle.state = StatusCode.Success - - def CheckError(particle, fieldset, time): - if particle.state >= 50: # This captures all Errors - particle.delete() - - class ArgoParticle(JITParticle): - cycle_phase = Variable("cycle_phase", dtype=np.int32, initial=0.0) - cycle_age = Variable("cycle_age", dtype=np.float32, initial=0.0) - drift_age = Variable("drift_age", dtype=np.float32, initial=0.0) - salinity = Variable("salinity", initial=np.nan) - temperature = Variable("temperature", initial=np.nan) - - # initialize argo floats - lon = [] - lat = [] - for i in range(len(config.argo_deploylocations)): - lon.append(config.argo_deploylocations[i][0]) - lat.append(config.argo_deploylocations[i][1]) - time = argo_time - - # Create and execute argo particles - argoset = ParticleSet( - fieldset=fieldset, - pclass=ArgoParticle, - lon=lon, - lat=lat, - depth=np.repeat(fieldset.mindepth, len(time)), - time=time, - ) - argo_output_file = argoset.ParticleFile( - name=os.path.join("results", "Argos.zarr"), - outputdt=timedelta(minutes=5), - chunks=(1, 500), - ) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - argo_endtime = np.array( - ( - datetime.datetime.strptime( - config.requested_ship_time["start"], "%Y-%m-%dT%H:%M:%S" - ) - + timedelta(weeks=6) - ), - dtype="datetime64[ms]", - ) - - argoset.execute( - [ - ArgoVerticalMovement, - AdvectionRK4, - KeepAtSurface, - CheckError, - ], # list of kernels to be executed - endtime=min(fieldset_endtime, argo_endtime), - dt=timedelta(minutes=5), - output_file=argo_output_file, - ) - - -def create_argo_fieldset(config, data_dir: str): - """ - Create a fieldset from netcdf files for argo floats, return fieldset with negative depth values. - - :param config: The cruise configuration. - :returns: The fieldset. - """ - filenames = { - "U": os.path.join(data_dir, "argodata_UV.nc"), - "V": os.path.join(data_dir, "argodata_UV.nc"), - "S": os.path.join(data_dir, "argodata_S.nc"), - "T": os.path.join(data_dir, "argodata_T.nc"), - } - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} - dimensions = { - "lon": "longitude", - "lat": "latitude", - "time": "time", - "depth": "depth", - } - - # create the fieldset and set interpolation methods - fieldset = FieldSet.from_netcdf( - filenames, variables, dimensions, allow_time_extrapolation=False - ) - fieldset.T.interp_method = "linear_invdist_land_tracer" - for g in fieldset.gridset.grids: - if max(g.depth) > 0: - g.depth = -g.depth # make depth negative - fieldset.mindepth = -fieldset.U.depth[0] # uppermost layer in the hydrodynamic data - fieldset.add_constant("driftdepth", config.argo_characteristics["driftdepth"]) - fieldset.add_constant("maxdepth", config.argo_characteristics["maxdepth"]) - fieldset.add_constant( - "vertical_speed", config.argo_characteristics["vertical_speed"] - ) - fieldset.add_constant("cycle_days", config.argo_characteristics["cycle_days"]) - fieldset.add_constant("drift_days", config.argo_characteristics["drift_days"]) - return fieldset diff --git a/virtual_ship/drifter_deployments.py b/virtual_ship/drifter_deployments.py index 84961906..69836bc3 100644 --- a/virtual_ship/drifter_deployments.py +++ b/virtual_ship/drifter_deployments.py @@ -6,6 +6,7 @@ import numpy as np from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable + from .virtual_ship_configuration import VirtualShipConfiguration @@ -80,6 +81,7 @@ def create_drifter_fieldset(config, data_dir: str): Create a fieldset from netcdf files for drifters, returns fieldset with negative depth values. :param config: The cruise configuration. + :param data_dir: TODO :returns: The fieldset. """ filenames = { diff --git a/virtual_ship/instruments/argo.py b/virtual_ship/instruments/argo.py index 06f061dd..4eee3047 100644 --- a/virtual_ship/instruments/argo.py +++ b/virtual_ship/instruments/argo.py @@ -1,20 +1,26 @@ -from .location import Location +"""Argo instrument.""" + +import math from dataclasses import dataclass +from datetime import timedelta + +import numpy as np from parcels import ( - ParticleSet, - JITParticle, - Variable, - FieldSet, AdvectionRK4, + FieldSet, + JITParticle, + ParticleSet, StatusCode, + Variable, ) -import math -import numpy as np -from datetime import timedelta + +from .location import Location @dataclass class Argo: + """Configuration for a single Argo.""" + location: Location deployment_time: float @@ -117,7 +123,6 @@ def simulate_argos( :param cycle_days: TODO :param drift_days: TODO """ - lon = [argo.location.lon for argo in argos] lat = [argo.location.lat for argo in argos] time = [argo.deployment_time for argo in argos] diff --git a/virtual_ship/instruments/location.py b/virtual_ship/instruments/location.py index 5807485e..e9501386 100644 --- a/virtual_ship/instruments/location.py +++ b/virtual_ship/instruments/location.py @@ -1,15 +1,29 @@ +"""Location class.""" + from dataclasses import dataclass @dataclass(frozen=True) class Location: + """A location on earth.""" + latitude: float longitude: float @property def lat(self) -> float: + """ + Shorthand for latitude variable. + + :returns: Latitude variable. + """ return self.latitude @property def lon(self) -> float: + """ + Shorthand for longitude variable. + + :returns: Longitude variable. + """ return self.longitude diff --git a/virtual_ship/sailship.py b/virtual_ship/sailship.py index 2536c62b..4b3bcd0f 100644 --- a/virtual_ship/sailship.py +++ b/virtual_ship/sailship.py @@ -7,13 +7,13 @@ import pyproj from parcels import Field, FieldSet, JITParticle, ParticleSet, Variable from shapely.geometry import Point, Polygon -from .drifter_deployments import drifter_deployments -from .argo_deployments import argo_deployments -from .postprocess import postprocess + from .costs import costs -from .virtual_ship_configuration import VirtualShipConfiguration +from .drifter_deployments import drifter_deployments from .instruments.argo import Argo, simulate_argos from .instruments.location import Location +from .postprocess import postprocess +from .virtual_ship_configuration import VirtualShipConfiguration def sailship(config: VirtualShipConfiguration): @@ -21,7 +21,7 @@ def sailship(config: VirtualShipConfiguration): Use parcels to simulate the ship, take CTDs and measure ADCP and underwaydata. :param config: The cruise configuration. - :returns: drifter_time, argo_time, total_time + :raises NotImplementedError: TODO """ # Create fieldset and retreive final schip route as sample_lons and sample_lats fieldset = config.ctd_fieldset # create_fieldset(config, data_dir) @@ -290,6 +290,7 @@ def create_fieldset(config, data_dir: str): Create fieldset from netcdf files and adds bathymetry data for CTD cast, returns fieldset with negative depth values. :param config: The cruise configuration. + :param data_dir: TODO :returns: The fieldset. :raises ValueError: If downloaded data is not as expected. """ diff --git a/virtual_ship/virtual_ship_configuration.py b/virtual_ship/virtual_ship_configuration.py index 841691ab..0057700c 100644 --- a/virtual_ship/virtual_ship_configuration.py +++ b/virtual_ship/virtual_ship_configuration.py @@ -2,11 +2,10 @@ import datetime import json -import os from datetime import timedelta -from shapely.geometry import Point, Polygon from parcels import FieldSet +from shapely.geometry import Point, Polygon class VirtualShipConfiguration: @@ -27,6 +26,9 @@ def __init__( Initialize this object. :param json_file: Path to the JSON file to init from. + :param ctd_fieldset: Fieldset for CTD measurements. + :param drifter_fieldset: Fieldset for CTD measurements. + :param argo_fieldset: FIeldset for argo measurements. :raises ValueError: If JSON file not valid. """ self.ctd_fieldset = ctd_fieldset From 528d1b3a91a5f357ce1eb076b350f79b26380e6b Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 14:56:44 +0200 Subject: [PATCH 10/29] Make constants individual parameters for argos --- tests/{sensors => instruments}/test_argo.py | 26 ++++---- virtual_ship/instruments/argo.py | 70 ++++++++++----------- 2 files changed, 48 insertions(+), 48 deletions(-) rename tests/{sensors => instruments}/test_argo.py (56%) diff --git a/tests/sensors/test_argo.py b/tests/instruments/test_argo.py similarity index 56% rename from tests/sensors/test_argo.py rename to tests/instruments/test_argo.py index a9f63ca4..f295ef79 100644 --- a/tests/sensors/test_argo.py +++ b/tests/instruments/test_argo.py @@ -22,15 +22,19 @@ def test_simulate_argos() -> None: }, ) - argos = [Argo(location=Location(latitude=0, longitude=0), deployment_time=0)] + min_depth = -fieldset.U.depth[0] - simulate_argos( - argos=argos, - fieldset=fieldset, - out_file_name="test", - drift_depth=DRIFT_DEPTH, - max_depth=MAX_DEPTH, - verticle_speed=VERTICLE_SPEED, - cycle_days=CYCLE_DAYS, - drift_days=DRIFT_DAYS, - ) + argos = [ + Argo( + location=Location(latitude=0, longitude=0), + deployment_time=0, + min_depth=min_depth, + max_depth=MAX_DEPTH, + drift_depth=DRIFT_DEPTH, + vertical_speed=VERTICLE_SPEED, + cycle_days=CYCLE_DAYS, + drift_days=DRIFT_DAYS, + ) + ] + + simulate_argos(argos=argos, fieldset=fieldset, out_file_name="test") diff --git a/virtual_ship/instruments/argo.py b/virtual_ship/instruments/argo.py index 4eee3047..7057fccb 100644 --- a/virtual_ship/instruments/argo.py +++ b/virtual_ship/instruments/argo.py @@ -23,6 +23,12 @@ class Argo: location: Location deployment_time: float + min_depth: float + max_depth: float + drift_depth: float + vertical_speed: float + cycle_days: float + drift_days: float class _ArgoParticle(JITParticle): @@ -31,41 +37,46 @@ class _ArgoParticle(JITParticle): drift_age = Variable("drift_age", dtype=np.float32, initial=0.0) salinity = Variable("salinity", initial=np.nan) temperature = Variable("temperature", initial=np.nan) + min_depth = Variable("min_depth", dtype=np.float32) + max_depth = Variable("max_depth", dtype=np.float32) + drift_depth = Variable("drift_depth", dtype=np.float32) + vertical_speed = Variable("vertical_speed", dtype=np.float32) + cycle_days = Variable("cycle_days", dtype=np.int32) + drift_days = Variable("drift_days", dtype=np.int32) def _argo_vertical_movement(particle, fieldset, time): - if particle.cycle_phase == 0: - # Phase 0: Sinking with vertical_speed until depth is driftdepth + # Phase 0: Sinking with vertical_speed until depth is drift_depth particle_ddepth += ( # noqa See comment above about particle_* variables. - fieldset.vertical_speed * particle.dt + particle.vertical_speed * particle.dt ) - if particle.depth + particle_ddepth <= fieldset.driftdepth: - particle_ddepth = fieldset.driftdepth - particle.depth + if particle.depth + particle_ddepth <= particle.drift_depth: + particle_ddepth = particle.drift_depth - particle.depth particle.cycle_phase = 1 elif particle.cycle_phase == 1: # Phase 1: Drifting at depth for drifttime seconds particle.drift_age += particle.dt - if particle.drift_age >= fieldset.drift_days * 86400: + if particle.drift_age >= particle.drift_days * 86400: particle.drift_age = 0 # reset drift_age for next cycle particle.cycle_phase = 2 elif particle.cycle_phase == 2: - # Phase 2: Sinking further to maxdepth - particle_ddepth += fieldset.vertical_speed * particle.dt - if particle.depth + particle_ddepth <= fieldset.maxdepth: - particle_ddepth = fieldset.maxdepth - particle.depth + # Phase 2: Sinking further to max_depth + particle_ddepth += particle.vertical_speed * particle.dt + if particle.depth + particle_ddepth <= particle.max_depth: + particle_ddepth = particle.max_depth - particle.depth particle.cycle_phase = 3 elif particle.cycle_phase == 3: # Phase 3: Rising with vertical_speed until at surface - particle_ddepth -= fieldset.vertical_speed * particle.dt + particle_ddepth -= particle.vertical_speed * particle.dt particle.cycle_age += ( particle.dt ) # solve issue of not updating cycle_age during ascent - if particle.depth + particle_ddepth >= fieldset.min_depth: - particle_ddepth = fieldset.min_depth - particle.depth + if particle.depth + particle_ddepth >= particle.min_depth: + particle_ddepth = particle.min_depth - particle.depth particle.temperature = ( math.nan ) # reset temperature to NaN at end of sampling cycle @@ -81,7 +92,7 @@ def _argo_vertical_movement(particle, fieldset, time): elif particle.cycle_phase == 4: # Phase 4: Transmitting at surface until cycletime is reached - if particle.cycle_age > fieldset.cycle_days * 86400: + if particle.cycle_age > particle.cycle_days * 86400: particle.cycle_phase = 0 particle.cycle_age = 0 @@ -92,7 +103,7 @@ def _argo_vertical_movement(particle, fieldset, time): def _keep_at_surface(particle, fieldset, time): # Prevent error when float reaches surface if particle.state == StatusCode.ErrorThroughSurface: - particle.depth = fieldset.min_depth + particle.depth = particle.min_depth particle.state = StatusCode.Success @@ -105,11 +116,6 @@ def simulate_argos( argos: list[Argo], fieldset: FieldSet, out_file_name: str, - max_depth: float, - drift_depth: float, - verticle_speed: float, - cycle_days: float, - drift_days: float, ) -> None: """ Use parcels to simulate a set of argos in a fieldset. @@ -117,26 +123,25 @@ def simulate_argos( :param argos: The argos to simulate. :param fieldset: The fieldset to simulate the argos in. :param out_file_name: The file to write the results to. - :param max_depth: TODO - :param drift_depth: TODO - :param verticle_speed: TODO - :param cycle_days: TODO - :param drift_days: TODO """ lon = [argo.location.lon for argo in argos] lat = [argo.location.lat for argo in argos] time = [argo.deployment_time for argo in argos] - min_depth = -fieldset.U.depth[0] - # define the parcels simulation argoset = ParticleSet( fieldset=fieldset, pclass=_ArgoParticle, lon=lon, lat=lat, - depth=np.repeat(min_depth, len(argos)), + depth=[argo.min_depth for argo in argos], time=time, + min_depth=[argo.min_depth for argo in argos], + max_depth=[argo.max_depth for argo in argos], + drift_depth=[argo.drift_depth for argo in argos], + vertical_speed=[argo.vertical_speed for argo in argos], + cycle_days=[argo.cycle_days for argo in argos], + drift_days=[argo.drift_days for argo in argos], ) # define output file for the simulation @@ -149,15 +154,6 @@ def simulate_argos( # get time when the fieldset ends fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - # set constants on fieldset required by kernels. - # sadly we must change the fieldset parameter to pass this information - fieldset.add_constant("min_depth", min_depth) - fieldset.add_constant("maxdepth", max_depth) - fieldset.add_constant("driftdepth", drift_depth) - fieldset.add_constant("vertical_speed", verticle_speed) - fieldset.add_constant("cycle_days", cycle_days) - fieldset.add_constant("drift_days", drift_days) - # execute simulation argoset.execute( [ From 02ee54d53d689f9468a6175066d92da8074e9bd0 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 15:01:25 +0200 Subject: [PATCH 11/29] update sailship to use new argo function --- virtual_ship/sailship.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/virtual_ship/sailship.py b/virtual_ship/sailship.py index 4b3bcd0f..69b8a038 100644 --- a/virtual_ship/sailship.py +++ b/virtual_ship/sailship.py @@ -141,6 +141,8 @@ def SampleT(particle, fieldset, time): argo = 0 argos: list[Argo] = [] + ARGO_MIN_DEPTH = -config.argo_fieldset.U.depth[0] + # run the model for the length of the sample_lons list for i in range(len(sample_lons) - 1): @@ -229,6 +231,12 @@ def SampleT(particle, fieldset, time): longitude=config.argo_deploylocations[argo][1], ), deployment_time=total_time, + min_depth=ARGO_MIN_DEPTH, + max_depth=config.argo_characteristics["maxdepth"], + drift_depth=config.argo_characteristics["driftdepth"], + vertical_speed=config.argo_characteristics["vertical_speed"], + cycle_days=config.argo_characteristics["cycle_days"], + drift_days=config.argo_characteristics["drift_days"], ) ) argo += 1 @@ -267,11 +275,6 @@ def SampleT(particle, fieldset, time): argos=argos, fieldset=config.argo_fieldset, out_file_name=os.path.join("results", "Argos.zarr"), - max_depth=config.argo_characteristics["maxdepth"], - drift_depth=config.argo_characteristics["driftdepth"], - verticle_speed=config.argo_characteristics["vertical_speed"], - cycle_days=config.argo_characteristics["cycle_days"], - drift_days=config.argo_characteristics["drift_days"], ) # convert CTD data to CSV From a6947331795d5e6b463ea91992ce1c72b6610d3a Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 15:16:56 +0200 Subject: [PATCH 12/29] whitespace --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6f875b36..924a28fe 100644 --- a/.gitignore +++ b/.gitignore @@ -173,4 +173,4 @@ cython_debug/ # Auto generated by setuptools scm virtual_ship/_version_setup.py -.vscode/ \ No newline at end of file +.vscode/ From 635110e2b322e13395e915698399388678b2bd02 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 15:21:33 +0200 Subject: [PATCH 13/29] whitespace --- tests/sailship_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sailship_config.json b/tests/sailship_config.json index e406f7fc..23cf4c1a 100644 --- a/tests/sailship_config.json +++ b/tests/sailship_config.json @@ -37,4 +37,4 @@ "cycle_days" : 10, "drift_days": 9 } - } \ No newline at end of file + } From 72a83cd08b1432bf0d22f50ae90a06665ecaf8e2 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 15:23:31 +0200 Subject: [PATCH 14/29] comment --- virtual_ship/instruments/location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtual_ship/instruments/location.py b/virtual_ship/instruments/location.py index e9501386..bfb9b771 100644 --- a/virtual_ship/instruments/location.py +++ b/virtual_ship/instruments/location.py @@ -5,7 +5,7 @@ @dataclass(frozen=True) class Location: - """A location on earth.""" + """A location on a sphere.""" latitude: float longitude: float From ad173e2f0d3e67145e9aa83d9ed4941bc5a3bd90 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 15:46:37 +0200 Subject: [PATCH 15/29] Add running tests with codecov --- .github/workflows/codetools.yml | 12 ++++++++++++ pyproject.toml | 2 ++ tests.sh | 5 +++++ 3 files changed, 19 insertions(+) create mode 100755 tests.sh diff --git a/.github/workflows/codetools.yml b/.github/workflows/codetools.yml index b53a9d5b..b9eae9f4 100644 --- a/.github/workflows/codetools.yml +++ b/.github/workflows/codetools.yml @@ -31,3 +31,15 @@ jobs: run: black --diff --check ./$PACKAGE - name: isort run: isort --check-only --diff ./$PACKAGE + + tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5.1.0 + with: + python-version: 3.12 + - name: install + run: pip install ".[dev]" + - name: run_tests + run: pytest --cov=virtual_ship tests diff --git a/pyproject.toml b/pyproject.toml index 9a012b3e..93804ace 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,8 @@ dev = [ "pydocstyle == 6.3.0", "sort-all == 1.2.0", "pytest == 8.2.0", + "pytest-cov == 5.0.0", + "codecov == 2.1.13", ] [tool.isort] diff --git a/tests.sh b/tests.sh new file mode 100755 index 00000000..57bb72c6 --- /dev/null +++ b/tests.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +# Runs the tests and creates a code coverage report. + +pytest --cov=virtual_ship --cov-report=html tests \ No newline at end of file From 89f88a68710b9a435a15acdaefb6a2ceae938727 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 15:48:11 +0200 Subject: [PATCH 16/29] Add missing dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 93804ace..bfa4e2d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "parcels >= 3, < 4", "scipy >= 1, < 2", "xarray >= 2023, < 2024", + "cftime >= 1, < 2", ] [project.urls] From 49c7d506eaae48e67d415bb565aca5b0079fb5e4 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 15:53:13 +0200 Subject: [PATCH 17/29] Add missing dependencies --- pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index bfa4e2d7..6ae71371 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,15 @@ dependencies = [ "parcels >= 3, < 4", "scipy >= 1, < 2", "xarray >= 2023, < 2024", + # The packages below are parcels packages not included in their distribution requirements. + "cgen >= 2020, < 2021", + "dask >= 2024, < 2025", "cftime >= 1, < 2", + "psutil >= 1, < 2", + "netCDF4 >= 1, < 2", + "zarr >= 2, < 3", + "tqdm >= 4, < 5", + "pymbolic >= 2022, < 2023", ] [project.urls] From e566bdb908e009f89aa64274938106330a43e140 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 16:10:13 +0200 Subject: [PATCH 18/29] fix dependency? --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 6ae71371..290c3836 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "scipy >= 1, < 2", "xarray >= 2023, < 2024", # The packages below are parcels packages not included in their distribution requirements. + "numpy >= 1, < 2", "cgen >= 2020, < 2021", "dask >= 2024, < 2025", "cftime >= 1, < 2", From 9ad60081eb1f62c58522105544063299f550983d Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Fri, 24 May 2024 16:12:46 +0200 Subject: [PATCH 19/29] fix dependency? --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 290c3836..033d157a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ # The packages below are parcels packages not included in their distribution requirements. "numpy >= 1, < 2", "cgen >= 2020, < 2021", - "dask >= 2024, < 2025", + "dask >= 2023, < 2025", "cftime >= 1, < 2", "psutil >= 1, < 2", "netCDF4 >= 1, < 2", From c4d4555cf33cc18911bc1dcc84dfb950358057da Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Mon, 27 May 2024 17:05:32 +0200 Subject: [PATCH 20/29] Add missing newline Co-authored-by: Erik van Sebille --- tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.sh b/tests.sh index 57bb72c6..a6203336 100755 --- a/tests.sh +++ b/tests.sh @@ -2,4 +2,4 @@ # Runs the tests and creates a code coverage report. -pytest --cov=virtual_ship --cov-report=html tests \ No newline at end of file +pytest --cov=virtual_ship --cov-report=html tests From 87b040f91ec22010bbd47e18d8e1f82bd569502a Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Mon, 27 May 2024 17:08:59 +0200 Subject: [PATCH 21/29] Comments update Co-authored-by: Erik van Sebille --- virtual_ship/instruments/argo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtual_ship/instruments/argo.py b/virtual_ship/instruments/argo.py index 7057fccb..a4e862ab 100644 --- a/virtual_ship/instruments/argo.py +++ b/virtual_ship/instruments/argo.py @@ -118,7 +118,7 @@ def simulate_argos( out_file_name: str, ) -> None: """ - Use parcels to simulate a set of argos in a fieldset. + Use parcels to simulate a set of Argo floats in a fieldset. :param argos: The argos to simulate. :param fieldset: The fieldset to simulate the argos in. From 3e42563988ba15880d1d059ae8de640818cd0efa Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Mon, 27 May 2024 17:09:19 +0200 Subject: [PATCH 22/29] Docstring update Co-authored-by: Erik van Sebille --- virtual_ship/instruments/argo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtual_ship/instruments/argo.py b/virtual_ship/instruments/argo.py index a4e862ab..335b6a64 100644 --- a/virtual_ship/instruments/argo.py +++ b/virtual_ship/instruments/argo.py @@ -120,7 +120,7 @@ def simulate_argos( """ Use parcels to simulate a set of Argo floats in a fieldset. - :param argos: The argos to simulate. + :param argos: A list of Argo floats to simulate. :param fieldset: The fieldset to simulate the argos in. :param out_file_name: The file to write the results to. """ From 01ffb83896135ce0d25738afe57bd6571e1c1042 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Tue, 28 May 2024 14:33:13 +0200 Subject: [PATCH 23/29] Rename argos to argo floats --- tests/instruments/test_argo.py | 16 ++++--- tests/test_sailship.py | 4 +- virtual_ship/instruments/__init__.py | 6 +++ .../instruments/{argo.py => argo_float.py} | 44 +++++++++---------- virtual_ship/sailship.py | 18 ++++---- virtual_ship/virtual_ship_configuration.py | 8 ++-- 6 files changed, 52 insertions(+), 44 deletions(-) create mode 100644 virtual_ship/instruments/__init__.py rename virtual_ship/instruments/{argo.py => argo_float.py} (80%) diff --git a/tests/instruments/test_argo.py b/tests/instruments/test_argo.py index f295ef79..f82d108b 100644 --- a/tests/instruments/test_argo.py +++ b/tests/instruments/test_argo.py @@ -1,12 +1,12 @@ -"""Test the simulation of argos.""" +"""Test the simulation of Argo floats.""" -from virtual_ship.instruments.argo import simulate_argos, Argo -from virtual_ship.instruments.location import Location +from virtual_ship.instruments.argo_float import simulate_argo_floats, ArgoFloat +from virtual_ship.instruments import Location from parcels import FieldSet import numpy as np -def test_simulate_argos() -> None: +def test_simulate_argo_floats() -> None: DRIFT_DEPTH = -1000 MAX_DEPTH = -2000 VERTICLE_SPEED = -0.10 @@ -24,8 +24,8 @@ def test_simulate_argos() -> None: min_depth = -fieldset.U.depth[0] - argos = [ - Argo( + argo_floats = [ + ArgoFloat( location=Location(latitude=0, longitude=0), deployment_time=0, min_depth=min_depth, @@ -37,4 +37,6 @@ def test_simulate_argos() -> None: ) ] - simulate_argos(argos=argos, fieldset=fieldset, out_file_name="test") + simulate_argo_floats( + argo_floats=argo_floats, fieldset=fieldset, out_file_name="test" + ) diff --git a/tests/test_sailship.py b/tests/test_sailship.py index 845fe141..cc8f13bb 100644 --- a/tests/test_sailship.py +++ b/tests/test_sailship.py @@ -23,7 +23,7 @@ def test_sailship() -> None: {"lon": 0, "lat": 0}, ) - argo_fieldset = FieldSet.from_data( + argo_float_fieldset = FieldSet.from_data( {"U": 0, "V": 0, "T": 0, "S": 0}, { "lon": 0, @@ -36,7 +36,7 @@ def test_sailship() -> None: "sailship_config.json", ctd_fieldset=ctd_fieldset, drifter_fieldset=drifter_fieldset, - argo_fieldset=argo_fieldset, + argo_float_fieldset=argo_float_fieldset, ) sailship(config) diff --git a/virtual_ship/instruments/__init__.py b/virtual_ship/instruments/__init__.py new file mode 100644 index 00000000..283e7cba --- /dev/null +++ b/virtual_ship/instruments/__init__.py @@ -0,0 +1,6 @@ +"""Measurement instrument that can be used with Parcels.""" + +from . import argo_float +from .location import Location + +__all__ = ["Location", "argo_float"] diff --git a/virtual_ship/instruments/argo.py b/virtual_ship/instruments/argo_float.py similarity index 80% rename from virtual_ship/instruments/argo.py rename to virtual_ship/instruments/argo_float.py index 335b6a64..8ffa7658 100644 --- a/virtual_ship/instruments/argo.py +++ b/virtual_ship/instruments/argo_float.py @@ -1,4 +1,4 @@ -"""Argo instrument.""" +"""Argo float instrument.""" import math from dataclasses import dataclass @@ -18,8 +18,8 @@ @dataclass -class Argo: - """Configuration for a single Argo.""" +class ArgoFloat: + """Configuration for a single Argo float.""" location: Location deployment_time: float @@ -45,7 +45,7 @@ class _ArgoParticle(JITParticle): drift_days = Variable("drift_days", dtype=np.int32) -def _argo_vertical_movement(particle, fieldset, time): +def _argo_float_vertical_movement(particle, fieldset, time): if particle.cycle_phase == 0: # Phase 0: Sinking with vertical_speed until depth is drift_depth particle_ddepth += ( # noqa See comment above about particle_* variables. @@ -112,40 +112,40 @@ def _check_error(particle, fieldset, time): particle.delete() -def simulate_argos( - argos: list[Argo], +def simulate_argo_floats( + argo_floats: list[ArgoFloat], fieldset: FieldSet, out_file_name: str, ) -> None: """ Use parcels to simulate a set of Argo floats in a fieldset. - :param argos: A list of Argo floats to simulate. - :param fieldset: The fieldset to simulate the argos in. + :param argo_floats: A list of Argo floats to simulate. + :param fieldset: The fieldset to simulate the Argo floats in. :param out_file_name: The file to write the results to. """ - lon = [argo.location.lon for argo in argos] - lat = [argo.location.lat for argo in argos] - time = [argo.deployment_time for argo in argos] + lon = [argo.location.lon for argo in argo_floats] + lat = [argo.location.lat for argo in argo_floats] + time = [argo.deployment_time for argo in argo_floats] # define the parcels simulation - argoset = ParticleSet( + argo_float_fieldset = ParticleSet( fieldset=fieldset, pclass=_ArgoParticle, lon=lon, lat=lat, - depth=[argo.min_depth for argo in argos], + depth=[argo.min_depth for argo in argo_floats], time=time, - min_depth=[argo.min_depth for argo in argos], - max_depth=[argo.max_depth for argo in argos], - drift_depth=[argo.drift_depth for argo in argos], - vertical_speed=[argo.vertical_speed for argo in argos], - cycle_days=[argo.cycle_days for argo in argos], - drift_days=[argo.drift_days for argo in argos], + min_depth=[argo.min_depth for argo in argo_floats], + max_depth=[argo.max_depth for argo in argo_floats], + drift_depth=[argo.drift_depth for argo in argo_floats], + vertical_speed=[argo.vertical_speed for argo in argo_floats], + cycle_days=[argo.cycle_days for argo in argo_floats], + drift_days=[argo.drift_days for argo in argo_floats], ) # define output file for the simulation - out_file = argoset.ParticleFile( + out_file = argo_float_fieldset.ParticleFile( name=out_file_name, outputdt=timedelta(minutes=5), chunks=(1, 500), @@ -155,9 +155,9 @@ def simulate_argos( fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) # execute simulation - argoset.execute( + argo_float_fieldset.execute( [ - _argo_vertical_movement, + _argo_float_vertical_movement, AdvectionRK4, _keep_at_surface, _check_error, diff --git a/virtual_ship/sailship.py b/virtual_ship/sailship.py index 69b8a038..9343ab74 100644 --- a/virtual_ship/sailship.py +++ b/virtual_ship/sailship.py @@ -10,7 +10,7 @@ from .costs import costs from .drifter_deployments import drifter_deployments -from .instruments.argo import Argo, simulate_argos +from .instruments.argo_float import ArgoFloat, simulate_argo_floats from .instruments.location import Location from .postprocess import postprocess from .virtual_ship_configuration import VirtualShipConfiguration @@ -139,9 +139,9 @@ def SampleT(particle, fieldset, time): drifter = 0 drifter_time = [] argo = 0 - argos: list[Argo] = [] + argo_floats: list[ArgoFloat] = [] - ARGO_MIN_DEPTH = -config.argo_fieldset.U.depth[0] + ARGO_MIN_DEPTH = -config.argo_float_fieldset.U.depth[0] # run the model for the length of the sample_lons list for i in range(len(sample_lons) - 1): @@ -224,8 +224,8 @@ def SampleT(particle, fieldset, time): abs(sample_lons[i] - config.argo_deploylocations[argo][0]) < 0.01 and abs(sample_lats[i] - config.argo_deploylocations[argo][1]) < 0.01 ): - argos.append( - Argo( + argo_floats.append( + ArgoFloat( location=Location( latitude=config.argo_deploylocations[argo][0], longitude=config.argo_deploylocations[argo][1], @@ -271,10 +271,10 @@ def SampleT(particle, fieldset, time): drifter_deployments(config, drifter_time) # simulate argo deployments - simulate_argos( - argos=argos, - fieldset=config.argo_fieldset, - out_file_name=os.path.join("results", "Argos.zarr"), + simulate_argo_floats( + argo_floats=argo_floats, + fieldset=config.argo_float_fieldset, + out_file_name=os.path.join("results", "argo_floats.zarr"), ) # convert CTD data to CSV diff --git a/virtual_ship/virtual_ship_configuration.py b/virtual_ship/virtual_ship_configuration.py index 0057700c..f74251b7 100644 --- a/virtual_ship/virtual_ship_configuration.py +++ b/virtual_ship/virtual_ship_configuration.py @@ -13,14 +13,14 @@ class VirtualShipConfiguration: ctd_fieldset: FieldSet drifter_fieldset: FieldSet - argo_fieldset: FieldSet + argo_float_fieldset: FieldSet def __init__( self, json_file, ctd_fieldset: FieldSet, drifter_fieldset: FieldSet, - argo_fieldset: FieldSet, + argo_float_fieldset: FieldSet, ): """ Initialize this object. @@ -28,12 +28,12 @@ def __init__( :param json_file: Path to the JSON file to init from. :param ctd_fieldset: Fieldset for CTD measurements. :param drifter_fieldset: Fieldset for CTD measurements. - :param argo_fieldset: FIeldset for argo measurements. + :param argo_float_fieldset: FIeldset for argo float measurements. :raises ValueError: If JSON file not valid. """ self.ctd_fieldset = ctd_fieldset self.drifter_fieldset = drifter_fieldset - self.argo_fieldset = argo_fieldset + self.argo_float_fieldset = argo_float_fieldset with open(json_file, "r") as file: json_input = json.loads(file.read()) From 1e1267f52fe8cd7c2de3ad6ebbe6550c40285b7e Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Tue, 28 May 2024 14:43:04 +0200 Subject: [PATCH 24/29] Create proper init.py --- virtual_ship/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/virtual_ship/__init__.py b/virtual_ship/__init__.py index d885347c..72890312 100644 --- a/virtual_ship/__init__.py +++ b/virtual_ship/__init__.py @@ -1 +1,6 @@ """Code for the Virtual Ship Classroom, where Marine Scientists can combine Copernicus Marine Data with an OceanParcels ship to go on a virtual expedition.""" + +import instruments +import sailship + +__all__ = ["instruments", "sailship"] From 3b65ecabead75c180ea12c97c9b824b85937db95 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Tue, 28 May 2024 15:16:44 +0200 Subject: [PATCH 25/29] make outputdf a parameters for argos --- tests/instruments/test_argo.py | 6 +++++- virtual_ship/__init__.py | 3 +-- virtual_ship/instruments/argo_float.py | 4 +++- virtual_ship/sailship.py | 1 + 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/instruments/test_argo.py b/tests/instruments/test_argo.py index f82d108b..f444c6e9 100644 --- a/tests/instruments/test_argo.py +++ b/tests/instruments/test_argo.py @@ -4,6 +4,7 @@ from virtual_ship.instruments import Location from parcels import FieldSet import numpy as np +from datetime import timedelta def test_simulate_argo_floats() -> None: @@ -38,5 +39,8 @@ def test_simulate_argo_floats() -> None: ] simulate_argo_floats( - argo_floats=argo_floats, fieldset=fieldset, out_file_name="test" + argo_floats=argo_floats, + fieldset=fieldset, + out_file_name="test", + outputdt=timedelta(minutes=5), ) diff --git a/virtual_ship/__init__.py b/virtual_ship/__init__.py index 72890312..bf152389 100644 --- a/virtual_ship/__init__.py +++ b/virtual_ship/__init__.py @@ -1,6 +1,5 @@ """Code for the Virtual Ship Classroom, where Marine Scientists can combine Copernicus Marine Data with an OceanParcels ship to go on a virtual expedition.""" -import instruments -import sailship +from . import instruments, sailship __all__ = ["instruments", "sailship"] diff --git a/virtual_ship/instruments/argo_float.py b/virtual_ship/instruments/argo_float.py index 8ffa7658..ae794228 100644 --- a/virtual_ship/instruments/argo_float.py +++ b/virtual_ship/instruments/argo_float.py @@ -116,6 +116,7 @@ def simulate_argo_floats( argo_floats: list[ArgoFloat], fieldset: FieldSet, out_file_name: str, + outputdt: timedelta, ) -> None: """ Use parcels to simulate a set of Argo floats in a fieldset. @@ -123,6 +124,7 @@ def simulate_argo_floats( :param argo_floats: A list of Argo floats to simulate. :param fieldset: The fieldset to simulate the Argo floats in. :param out_file_name: The file to write the results to. + :param outputdt: Interval which dictates the update frequency of file output during simulation """ lon = [argo.location.lon for argo in argo_floats] lat = [argo.location.lat for argo in argo_floats] @@ -163,6 +165,6 @@ def simulate_argo_floats( _check_error, ], endtime=fieldset_endtime, - dt=timedelta(minutes=5), + dt=outputdt, output_file=out_file, ) diff --git a/virtual_ship/sailship.py b/virtual_ship/sailship.py index 9343ab74..51238069 100644 --- a/virtual_ship/sailship.py +++ b/virtual_ship/sailship.py @@ -275,6 +275,7 @@ def SampleT(particle, fieldset, time): argo_floats=argo_floats, fieldset=config.argo_float_fieldset, out_file_name=os.path.join("results", "argo_floats.zarr"), + outputdt=timedelta(minutes=5), ) # convert CTD data to CSV From 36ead1bba3af0a4ab47bef5bc81e979bd8693846 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Tue, 28 May 2024 16:24:46 +0200 Subject: [PATCH 26/29] Minor docstring update --- virtual_ship/instruments/location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtual_ship/instruments/location.py b/virtual_ship/instruments/location.py index bfb9b771..edeb4efc 100644 --- a/virtual_ship/instruments/location.py +++ b/virtual_ship/instruments/location.py @@ -1,4 +1,4 @@ -"""Location class.""" +"""Location class. See class description.""" from dataclasses import dataclass From 5186d7e050842592a7f9d6490c4ea019b687f4b3 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Tue, 28 May 2024 16:34:42 +0200 Subject: [PATCH 27/29] minor variable renaming --- virtual_ship/instruments/argo_float.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/virtual_ship/instruments/argo_float.py b/virtual_ship/instruments/argo_float.py index ae794228..3c753cfe 100644 --- a/virtual_ship/instruments/argo_float.py +++ b/virtual_ship/instruments/argo_float.py @@ -131,7 +131,7 @@ def simulate_argo_floats( time = [argo.deployment_time for argo in argo_floats] # define the parcels simulation - argo_float_fieldset = ParticleSet( + argo_float_particleset = ParticleSet( fieldset=fieldset, pclass=_ArgoParticle, lon=lon, @@ -147,7 +147,7 @@ def simulate_argo_floats( ) # define output file for the simulation - out_file = argo_float_fieldset.ParticleFile( + out_file = argo_float_particleset.ParticleFile( name=out_file_name, outputdt=timedelta(minutes=5), chunks=(1, 500), @@ -157,7 +157,7 @@ def simulate_argo_floats( fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) # execute simulation - argo_float_fieldset.execute( + argo_float_particleset.execute( [ _argo_float_vertical_movement, AdvectionRK4, From 8bb7ee2de7a178676f9f40131caf34d110098c71 Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Tue, 28 May 2024 17:29:10 +0200 Subject: [PATCH 28/29] Run codetools on tests --- .github/workflows/codetools.yml | 7 ++++--- codetools.sh | 7 ++++--- tests/instruments/test_argo.py | 10 ++++++---- tests/test_sailship.py | 7 ++++--- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/codetools.yml b/.github/workflows/codetools.yml index b9eae9f4..7027fec2 100644 --- a/.github/workflows/codetools.yml +++ b/.github/workflows/codetools.yml @@ -4,6 +4,7 @@ on: [push, pull_request] env: PACKAGE: virtual_ship + TESTS: tests jobs: codetools: @@ -19,7 +20,7 @@ jobs: - name: install run: pip install ".[dev]" - name: flake8 - run: flake8 ./$PACKAGE + run: flake8 ./$PACKAGE ./$TESTS - name: pydocstyle run: pydocstyle ./$PACKAGE - name: sort-all @@ -28,9 +29,9 @@ jobs: [[ -z $(git status -s) ]] git checkout -- . - name: black - run: black --diff --check ./$PACKAGE + run: black --diff --check ./$PACKAGE ./$TESTS - name: isort - run: isort --check-only --diff ./$PACKAGE + run: isort --check-only --diff ./$PACKAGE ./$TESTS tests: runs-on: ubuntu-20.04 diff --git a/codetools.sh b/codetools.sh index 960a0eff..a359c5d6 100755 --- a/codetools.sh +++ b/codetools.sh @@ -9,11 +9,12 @@ set -e cd "$(dirname "$0")" PACKAGE=virtual_ship +TESTS=tests echo "--------------" echo "flake8" echo "--------------" -flake8 ./$PACKAGE +flake8 ./$PACKAGE ./$TESTS # darglint is ran as a plugin for flake8. echo "--------------" @@ -29,10 +30,10 @@ find ./$PACKAGE -type f -name '__init__.py' -print0 | xargs -0 sort-all echo "--------------" echo "black" echo "--------------" -black ./$PACKAGE +black ./$PACKAGE ./$TESTS echo "--------------" echo "isort" echo "--------------" -isort ./$PACKAGE +isort ./$PACKAGE ./$TESTS diff --git a/tests/instruments/test_argo.py b/tests/instruments/test_argo.py index f444c6e9..349ed051 100644 --- a/tests/instruments/test_argo.py +++ b/tests/instruments/test_argo.py @@ -1,11 +1,13 @@ """Test the simulation of Argo floats.""" -from virtual_ship.instruments.argo_float import simulate_argo_floats, ArgoFloat -from virtual_ship.instruments import Location -from parcels import FieldSet -import numpy as np from datetime import timedelta +import numpy as np +from parcels import FieldSet + +from virtual_ship.instruments import Location +from virtual_ship.instruments.argo_float import ArgoFloat, simulate_argo_floats + def test_simulate_argo_floats() -> None: DRIFT_DEPTH = -1000 diff --git a/tests/test_sailship.py b/tests/test_sailship.py index cc8f13bb..afd8a95f 100644 --- a/tests/test_sailship.py +++ b/tests/test_sailship.py @@ -1,9 +1,10 @@ """Performs a complete cruise with virtual ship.""" -from virtual_ship.virtual_ship_configuration import VirtualShipConfiguration -from virtual_ship.sailship import sailship -from parcels import FieldSet import numpy as np +from parcels import FieldSet + +from virtual_ship.sailship import sailship +from virtual_ship.virtual_ship_configuration import VirtualShipConfiguration def test_sailship() -> None: From 1cf2082233b4c16a0f97efaccb0e5b3082fcdeef Mon Sep 17 00:00:00 2001 From: Aart Stuurman Date: Wed, 29 May 2024 16:57:30 +0200 Subject: [PATCH 29/29] Use new parcels particle api --- virtual_ship/instruments/argo_float.py | 27 ++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/virtual_ship/instruments/argo_float.py b/virtual_ship/instruments/argo_float.py index 3c753cfe..d9e7d50e 100644 --- a/virtual_ship/instruments/argo_float.py +++ b/virtual_ship/instruments/argo_float.py @@ -31,18 +31,21 @@ class ArgoFloat: drift_days: float -class _ArgoParticle(JITParticle): - cycle_phase = Variable("cycle_phase", dtype=np.int32, initial=0.0) - cycle_age = Variable("cycle_age", dtype=np.float32, initial=0.0) - drift_age = Variable("drift_age", dtype=np.float32, initial=0.0) - salinity = Variable("salinity", initial=np.nan) - temperature = Variable("temperature", initial=np.nan) - min_depth = Variable("min_depth", dtype=np.float32) - max_depth = Variable("max_depth", dtype=np.float32) - drift_depth = Variable("drift_depth", dtype=np.float32) - vertical_speed = Variable("vertical_speed", dtype=np.float32) - cycle_days = Variable("cycle_days", dtype=np.int32) - drift_days = Variable("drift_days", dtype=np.int32) +_ArgoParticle = JITParticle.add_variables( + [ + Variable("cycle_phase", dtype=np.int32, initial=0.0), + Variable("cycle_age", dtype=np.float32, initial=0.0), + Variable("drift_age", dtype=np.float32, initial=0.0), + Variable("salinity", initial=np.nan), + Variable("temperature", initial=np.nan), + Variable("min_depth", dtype=np.float32), + Variable("max_depth", dtype=np.float32), + Variable("drift_depth", dtype=np.float32), + Variable("vertical_speed", dtype=np.float32), + Variable("cycle_days", dtype=np.int32), + Variable("drift_days", dtype=np.int32), + ] +) def _argo_float_vertical_movement(particle, fieldset, time):