From 3b909fcfe17d2b03947ec191e18318413f7af123 Mon Sep 17 00:00:00 2001 From: Martin Vonk Date: Wed, 2 Aug 2023 13:49:08 +0200 Subject: [PATCH 01/22] clearer marking per ytick for data_availability --- pastastore/plotting.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pastastore/plotting.py b/pastastore/plotting.py index 1f4aa1a..a5bee2c 100644 --- a/pastastore/plotting.py +++ b/pastastore/plotting.py @@ -397,20 +397,30 @@ def _data_availability( linewidth=0, rasterized=True, ) + # make a colorbar in an ax on the # right side, then set the current axes to ax again cb = fig.colorbar(pc, ax=ax, cax=cax, extend="both") cb.set_ticks(bounds) cb.ax.set_yticklabels(labels) cb.ax.minorticks_off() + if set_yticks: - ax.set_yticks(np.arange(0.5, len(series) + 0.5)) + ax.set_yticks(np.arange(0.5, len(series) + 0.5), major=True) + ax.set_yticks(np.arange(0, len(series) + 1), minor=True) if names is None: names = [s.name for s in series] ax.set_yticklabels(names) + + for tick in ax.yaxis.get_major_ticks(): # don't show major ytick marker + tick.tick1line.set_visible(False) + + ax.grid(True, which="minor", axis="y") + ax.grid(True, which="major", axis="x") + else: ax.set_ylabel("Timeseries (-)") - ax.grid(True) + ax.grid(True, which="both") return ax From 0c2805341b5e9d584df2d6f2f49360a8e2809e54 Mon Sep 17 00:00:00 2001 From: Martin Vonk Date: Wed, 2 Aug 2023 13:49:08 +0200 Subject: [PATCH 02/22] clearer markers for yticks in data_availability plot --- pastastore/plotting.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pastastore/plotting.py b/pastastore/plotting.py index 1f4aa1a..38cdcf5 100644 --- a/pastastore/plotting.py +++ b/pastastore/plotting.py @@ -397,20 +397,30 @@ def _data_availability( linewidth=0, rasterized=True, ) + # make a colorbar in an ax on the # right side, then set the current axes to ax again cb = fig.colorbar(pc, ax=ax, cax=cax, extend="both") cb.set_ticks(bounds) cb.ax.set_yticklabels(labels) cb.ax.minorticks_off() + if set_yticks: - ax.set_yticks(np.arange(0.5, len(series) + 0.5)) + ax.set_yticks(np.arange(0.5, len(series) + 0.5), minor=False) + ax.set_yticks(np.arange(0, len(series) + 1), minor=True) if names is None: names = [s.name for s in series] ax.set_yticklabels(names) + + for tick in ax.yaxis.get_major_ticks(): # don't show major ytick marker + tick.tick1line.set_visible(False) + + ax.grid(True, which="minor", axis="y") + ax.grid(True, which="major", axis="x") + else: ax.set_ylabel("Timeseries (-)") - ax.grid(True) + ax.grid(True, which="both") return ax From a20ba0e45626221832fc238b611f163f4ab89cf4 Mon Sep 17 00:00:00 2001 From: Martin Vonk Date: Wed, 2 Aug 2023 13:59:08 +0200 Subject: [PATCH 03/22] Update plotting.py alles ging mis.. --- pastastore/plotting.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pastastore/plotting.py b/pastastore/plotting.py index 93bf4e5..10a5db4 100644 --- a/pastastore/plotting.py +++ b/pastastore/plotting.py @@ -418,12 +418,6 @@ def _data_availability( ax.grid(True, which="minor", axis="y") ax.grid(True, which="major", axis="x") - for tick in ax.yaxis.get_major_ticks(): # don't show major ytick marker - tick.tick1line.set_visible(False) - - ax.grid(True, which="minor", axis="y") - ax.grid(True, which="major", axis="x") - else: ax.set_ylabel("Timeseries (-)") ax.grid(True, which="both") From 5e807a2cf3c4ed93472375a3a6b6a6082e23f3b9 Mon Sep 17 00:00:00 2001 From: Martin Vonk Date: Wed, 16 Aug 2023 11:03:55 +0200 Subject: [PATCH 04/22] check _d and _a for steptrend and linear trend should also fix #98 --- pastastore/util.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/pastastore/util.py b/pastastore/util.py index 70ab042..d9bb751 100644 --- a/pastastore/util.py +++ b/pastastore/util.py @@ -704,6 +704,8 @@ def frontiers_checks( ) else: tmem = ml.get_response_tmax(sm_name) + if tmem is None: # no rfunc in stressmodel + tmem = 0 check_tmem_passed = tmem < len_oseries_calib / 2 checks.loc[f"calib_period > 2*t_mem_95%: {sm_name}", :] = ( tmem, @@ -736,23 +738,30 @@ def frontiers_checks( "(unit head)/(unit well stress)", check_gain_passed, ) + continue + elif sm._name == "LinearTrend": + gain = ml.parameters.loc[f"{sm_name}_a", "optimal"] + gain_std = ml.parameters.loc[f"{sm_name}_a", "stderr"] + elif sm._name == "StepTrend": + gain = ml.parameters.loc[f"{sm_name}_d", "optimal"] + gain_std = ml.parameters.loc[f"{sm_name}_d", "stderr"] else: gain = ml.parameters.loc[f"{sm_name}_A", "optimal"] gain_std = ml.parameters.loc[f"{sm_name}_A", "stderr"] - if gain_std is None: - gain_std = np.nan - check_gain_passed = pd.NA - elif np.isnan(gain_std): - check_gain_passed = pd.NA - else: - check_gain_passed = np.abs(gain) > 2 * gain_std + + if gain_std is None: + gain_std = np.nan + check_gain_passed = pd.NA + elif np.isnan(gain_std): + check_gain_passed = pd.NA + else: check_gain_passed = np.abs(gain) > 2 * gain_std - checks.loc[f"gain > 2*std: {sm_name}", :] = ( - gain, - 2 * gain_std, - "(unit head)/(unit well stress)", - check_gain_passed, - ) + checks.loc[f"gain > 2*std: {sm_name}", :] = ( + gain, + 2 * gain_std, + "(unit head)/(unit well stress)", + check_gain_passed, + ) # Check 5 - Parameter Bounds if check5_parambounds: From 43d0ef4a4e38fc2ee57cbc60c0d944339855a988 Mon Sep 17 00:00:00 2001 From: Martin Vonk Date: Mon, 28 Aug 2023 16:44:41 +0200 Subject: [PATCH 05/22] steptrend is named stepmodel --- pastastore/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pastastore/util.py b/pastastore/util.py index d9bb751..48d477a 100644 --- a/pastastore/util.py +++ b/pastastore/util.py @@ -742,7 +742,7 @@ def frontiers_checks( elif sm._name == "LinearTrend": gain = ml.parameters.loc[f"{sm_name}_a", "optimal"] gain_std = ml.parameters.loc[f"{sm_name}_a", "stderr"] - elif sm._name == "StepTrend": + elif sm._name == "StepModel": gain = ml.parameters.loc[f"{sm_name}_d", "optimal"] gain_std = ml.parameters.loc[f"{sm_name}_d", "stderr"] else: From 3b8bb0ba0504ca2b4a1a6145409b73feba96d19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 7 Sep 2023 14:39:45 +0200 Subject: [PATCH 06/22] add apply func #99 + two fixes - add apply function - add test - import from tqdm.auto fixes - add modelnames kwarg to modelstat plot - support stress as dictionary in get_model_timeseries_names --- pastastore/base.py | 2 +- pastastore/plotting.py | 7 ++++++- pastastore/store.py | 19 +++++++++++++++++-- pastastore/util.py | 2 +- tests/test_003_pastastore.py | 8 ++++++++ 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/pastastore/base.py b/pastastore/base.py index ecf5517..c53825c 100644 --- a/pastastore/base.py +++ b/pastastore/base.py @@ -11,7 +11,7 @@ from numpy import isin from packaging.version import parse as parse_version from pastas.io.pas import PastasEncoder -from tqdm import tqdm +from tqdm.auto import tqdm from .util import ItemInLibraryException, _custom_warning, validate_names from .version import PASTAS_LEQ_022 diff --git a/pastastore/plotting.py b/pastastore/plotting.py index 10a5db4..49669a3 100644 --- a/pastastore/plotting.py +++ b/pastastore/plotting.py @@ -723,6 +723,7 @@ def models( def modelstat( self, statistic, + modelnames=None, label=True, adjust=False, cmap="viridis", @@ -739,6 +740,8 @@ def modelstat( ---------- statistic: str name of the statistic, e.g. "evp" or "aic" + modelnames : list of str, optional + list of modelnames to include label: bool, optional label points, by default True adjust: bool, optional @@ -768,7 +771,9 @@ def modelstat( -------- self.add_background_map """ - statsdf = self.pstore.get_statistics([statistic], progressbar=False).to_frame() + statsdf = self.pstore.get_statistics( + [statistic], modelnames=modelnames, progressbar=False + ).to_frame() statsdf["oseries"] = [ self.pstore.get_models(m, return_dict=True)["oseries"]["name"] diff --git a/pastastore/store.py b/pastastore/store.py index c39fee1..243d9ed 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -8,7 +8,7 @@ import pastas as ps from packaging.version import parse as parse_version from pastas.io.pas import pastas_hook -from tqdm import tqdm +from tqdm.auto import tqdm from .plotting import Maps, Plots from .util import _custom_warning @@ -983,9 +983,24 @@ def get_model_timeseries_names( structure.loc[mlnam, pnam] = 1 structure.loc[mlnam, enam] = 1 elif "stress" in sm: - for s in sm["stress"]: + smstress = sm["stress"] + if isinstance(smstress, dict): + smstress = [smstress] + for s in smstress: structure.loc[mlnam, s["name"]] = 1 if dropna: return structure.dropna(how="all", axis=1) else: return structure + + def apply(self, libname, func, names=None, progressbar=True): + names = self.conn._parse_names(names, libname) + result = [] + if libname not in ("oseries", "stresses", "models"): + raise ValueError( + "'libname' must be one of ['oseries', 'stresses', 'models']!" + ) + getter = getattr(self.conn, f"get_{libname}") + for n in tqdm(names) if progressbar else names: + result.append(func(getter(n))) + return result diff --git a/pastastore/util.py b/pastastore/util.py index 48d477a..1c28f5a 100644 --- a/pastastore/util.py +++ b/pastastore/util.py @@ -6,7 +6,7 @@ from numpy.lib._iotools import NameValidator from pandas.testing import assert_series_equal from pastas.stats.tests import runs_test, stoffer_toloi -from tqdm import tqdm +from tqdm.auto import tqdm from .version import PASTAS_LEQ_022 diff --git a/tests/test_003_pastastore.py b/tests/test_003_pastastore.py index a7d398b..bdd430b 100644 --- a/tests/test_003_pastastore.py +++ b/tests/test_003_pastastore.py @@ -188,6 +188,14 @@ def test_solve_models_and_get_stats(request, pstore): assert stats.index.size == 2 +@pytest.mark.dependency() +def test_apply(request, pstore): + depends(request, [f"test_solve_models_and_get_stats[{pstore.type}]"]) + func = lambda ml: ml.parameters.loc["recharge_A", "optimal"] + result = pstore.apply("models", func) + assert len(result) == 2 + + @pytest.mark.dependency() def test_save_and_load_model(request, pstore): ml = pstore.create_model("oseries2") From 15f3bcbb76f53890035269517615140474046eb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 7 Sep 2023 14:44:39 +0200 Subject: [PATCH 07/22] add docstring --- pastastore/store.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pastastore/store.py b/pastastore/store.py index 243d9ed..ea57f4e 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -994,6 +994,27 @@ def get_model_timeseries_names( return structure def apply(self, libname, func, names=None, progressbar=True): + """Apply function to items in library. + + Supported libraries are oseries, stresses, and models. + + Parameters + ---------- + libname : str + library name, supports "oseries", "stresses" and "models" + func : callable + function that accepts items from one of the supported libraries as input + names : str, list of str, optional + apply function to these names, by default None which loops over all stored + items in library + progressbar : bool, optional + show progressbar, by default True + + Returns + ------- + list + list of results of func + """ names = self.conn._parse_names(names, libname) result = [] if libname not in ("oseries", "stresses", "models"): From 2c919a05b6adfb3ce9e3786e48dc6b61012106ec Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 7 Sep 2023 15:40:44 +0200 Subject: [PATCH 08/22] first attempt to add signatures to pastastore --- pastastore/store.py | 79 ++++++++++++++++++++++++++++++++++-- tests/test_003_pastastore.py | 7 ++++ 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/pastastore/store.py b/pastastore/store.py index c39fee1..bfb7ad9 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -300,6 +300,79 @@ def get_nearest_stresses( data = pd.concat([data, series], axis=0) return data + def get_signatures( + self, + signatures=None, + names=None, + libname="oseries", + progressbar=False, + ignore_errors=False, + ): + """Get groundwater signatures. NaN-values are returned when the + signature could not be computed. + + Parameters + ---------- + signatures : list of str, optional + list of groundwater signatures to compute, if None all groundwater + signatures in ps.stats.signatures.__all__ are used, by default None + names : str, list of str, or None, optional + names of the time series, by default None which + uses all the time series in the library + libname : str + name of the library containing the time series + ('oseries' or 'stresses'), by default "oseries" + progressbar : bool, optional + show progressbar, by default False + ignore_errors : bool, optional + ignore errors when True, i.e. when non-existent timeseries is + encountered in names, by default False + + Returns + ------- + sign : pandas.DataFrame + DataFrame containing the signatures (columns) per time series (rows) + """ + names = self.conn._parse_names(names, libname=libname) + + if signatures is None: + signatures = ps.stats.signatures.__all__.copy() + + # create dataframe for results + sign = pd.DataFrame(index=names, columns=signatures, data=np.nan) + + # loop through oseries names + desc = "Get groundwater signatures" + for name in tqdm(names, desc=desc) if progressbar else names: + try: + if libname == "oseries": + s = self.conn.get_oseries(name) + else: + s = self.conn.get_stresses(name) + except Exception as e: + if ignore_errors: + sign.loc[name, :] = np.nan + continue + else: + raise e + + try: + l = ps.stats.signatures.summary(s.squeeze(), signatures) + except Exception as e: + if ignore_errors: + l = [] + for sign in signatures: + try: + sign_val = getattr(ps.stats.signatures, sign)(s.squeeze()) + except: + sign_val = np.nan + l.append(sign_val) + else: + raise e + sign.loc[name, signatures] = l + + return sign + def get_tmin_tmax(self, libname, names=None, progressbar=False): """Get tmin and tmax for time series. @@ -428,13 +501,13 @@ def get_statistics( modelnames = self.conn._parse_names(modelnames, libname="models") - # create dataframe for results - s = pd.DataFrame(index=modelnames, columns=statistics, data=np.nan) - # if statistics is str if isinstance(statistics, str): statistics = [statistics] + # create dataframe for results + s = pd.DataFrame(index=modelnames, columns=statistics, data=np.nan) + # loop through model names desc = "Get model statistics" for mlname in tqdm(modelnames, desc=desc) if progressbar else modelnames: diff --git a/tests/test_003_pastastore.py b/tests/test_003_pastastore.py index a7d398b..cdb1a1a 100644 --- a/tests/test_003_pastastore.py +++ b/tests/test_003_pastastore.py @@ -172,6 +172,13 @@ def test_get_parameters(request, pstore): assert p.isna().sum().sum() == 0 +@pytest.mark.dependency() +def test_get_signatures(request, pstore): + depends(request, [f"test_create_models[{pstore.type}]"]) + s = pstore.get_signatures(progressbar=False) + assert s.shape[1] == len(ps.stats.signatures.__all__) + + @pytest.mark.dependency() def test_iter_models(request, pstore): depends(request, [f"test_create_models[{pstore.type}]"]) From c006bce8902fb135dfd67da67865cc96c93609d0 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Thu, 7 Sep 2023 15:48:26 +0200 Subject: [PATCH 09/22] fix variable name --- pastastore/store.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pastastore/store.py b/pastastore/store.py index bfb7ad9..95d23d6 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -361,9 +361,11 @@ def get_signatures( except Exception as e: if ignore_errors: l = [] - for sign in signatures: + for signature in signatures: try: - sign_val = getattr(ps.stats.signatures, sign)(s.squeeze()) + sign_val = getattr(ps.stats.signatures, signature)( + s.squeeze() + ) except: sign_val = np.nan l.append(sign_val) From c0d9b8b7fc60fe56fe477cff9dbfa1caf7ca09f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Sun, 10 Sep 2023 15:00:21 +0200 Subject: [PATCH 10/22] add default Connector add some type hints make imports more explicit --- docs/conf.py | 1 - pastastore/__init__.py | 8 ++++---- pastastore/base.py | 4 ++-- pastastore/connectors.py | 4 ++-- pastastore/store.py | 20 +++++++++++++------- pastastore/util.py | 2 +- pastastore/version.py | 2 +- pastastore/yaml_interface.py | 6 +++--- 8 files changed, 26 insertions(+), 21 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index cb90b72..1da1d1a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -125,7 +125,6 @@ # # html_sidebars = {} - # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. diff --git a/pastastore/__init__.py b/pastastore/__init__.py index 1ae7869..57aad16 100644 --- a/pastastore/__init__.py +++ b/pastastore/__init__.py @@ -1,10 +1,10 @@ -from . import connectors, util -from .connectors import ( +from pastastore import connectors, util +from pastastore.connectors import ( ArcticConnector, ArcticDBConnector, DictConnector, PasConnector, PystoreConnector, ) -from .store import PastaStore -from .version import __version__ +from pastastore.store import PastaStore +from pastastore.version import __version__ diff --git a/pastastore/base.py b/pastastore/base.py index c53825c..ffe98c8 100644 --- a/pastastore/base.py +++ b/pastastore/base.py @@ -13,8 +13,8 @@ from pastas.io.pas import PastasEncoder from tqdm.auto import tqdm -from .util import ItemInLibraryException, _custom_warning, validate_names -from .version import PASTAS_LEQ_022 +from pastastore.util import ItemInLibraryException, _custom_warning, validate_names +from pastastore.version import PASTAS_LEQ_022 FrameorSeriesUnion = Union[pd.DataFrame, pd.Series] warnings.showwarning = _custom_warning diff --git a/pastastore/connectors.py b/pastastore/connectors.py index 146f3bb..465be38 100644 --- a/pastastore/connectors.py +++ b/pastastore/connectors.py @@ -8,8 +8,8 @@ import pandas as pd from pastas.io.pas import PastasEncoder, pastas_hook -from .base import BaseConnector, ConnectorUtil, ModelAccessor -from .util import _custom_warning +from pastastore.base import BaseConnector, ConnectorUtil, ModelAccessor +from pastastore.util import _custom_warning FrameorSeriesUnion = Union[pd.DataFrame, pd.Series] warnings.showwarning = _custom_warning diff --git a/pastastore/store.py b/pastastore/store.py index ea57f4e..15268d2 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -10,9 +10,11 @@ from pastas.io.pas import pastas_hook from tqdm.auto import tqdm -from .plotting import Maps, Plots -from .util import _custom_warning -from .yaml_interface import PastastoreYAML +from pastastore.base import BaseConnector +from pastastore.connectors import DictConnector +from pastastore.plotting import Maps, Plots +from pastastore.util import _custom_warning +from pastastore.yaml_interface import PastastoreYAML FrameorSeriesUnion = Union[pd.DataFrame, pd.Series] warnings.showwarning = _custom_warning @@ -38,14 +40,18 @@ class PastaStore: name of the PastaStore, by default takes the name of the Connector object """ - def __init__(self, connector, name: str = None): + def __init__( + self, + connector: BaseConnector = DictConnector("pastas_db"), + name: Optional[str] = None, + ): """Initialize PastaStore for managing pastas time series and models. Parameters ---------- - connector : Connector object - object that provides the interface to the - database + connector : Connector object, optional + object that provides the connection to the database. Default is + DictConnector which does not store data on disk. name : str, optional name of the PastaStore, if not provided uses the Connector name """ diff --git a/pastastore/util.py b/pastastore/util.py index 1c28f5a..313c039 100644 --- a/pastastore/util.py +++ b/pastastore/util.py @@ -8,7 +8,7 @@ from pastas.stats.tests import runs_test, stoffer_toloi from tqdm.auto import tqdm -from .version import PASTAS_LEQ_022 +from pastastore.version import PASTAS_LEQ_022 def _custom_warning(message, category=UserWarning, filename="", lineno=-1, *args): diff --git a/pastastore/version.py b/pastastore/version.py index 3afd6c0..04980b4 100644 --- a/pastastore/version.py +++ b/pastastore/version.py @@ -4,4 +4,4 @@ PASTAS_VERSION = parse_version(ps.__version__) PASTAS_LEQ_022 = PASTAS_VERSION <= parse_version("0.22.0") -__version__ = "1.2.2" +__version__ = "1.3.0b" diff --git a/pastastore/yaml_interface.py b/pastastore/yaml_interface.py index 00a28a1..4d07c35 100644 --- a/pastastore/yaml_interface.py +++ b/pastastore/yaml_interface.py @@ -2,14 +2,14 @@ import logging import os from copy import deepcopy -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union import numpy as np import pandas as pd import pastas as ps import yaml -from .version import PASTAS_LEQ_022 +from pastastore.version import PASTAS_LEQ_022 ps.logger.setLevel("ERROR") @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -def _convert_dict_dtypes_for_yaml(d: Dict): +def _convert_dict_dtypes_for_yaml(d: Dict[str, Any]): """Internal method to convert dictionary values for storing in YAML format. Parameters From fb7523a9ff165b47c1fd95c613080bbe9b391880 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Tue, 17 Oct 2023 16:58:22 +0200 Subject: [PATCH 11/22] fix for comment --- pastastore/store.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pastastore/store.py b/pastastore/store.py index 95d23d6..4421aa8 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -330,7 +330,7 @@ def get_signatures( Returns ------- - sign : pandas.DataFrame + signatures_df : pandas.DataFrame DataFrame containing the signatures (columns) per time series (rows) """ names = self.conn._parse_names(names, libname=libname) @@ -339,7 +339,7 @@ def get_signatures( signatures = ps.stats.signatures.__all__.copy() # create dataframe for results - sign = pd.DataFrame(index=names, columns=signatures, data=np.nan) + signatures_df = pd.DataFrame(index=names, columns=signatures, data=np.nan) # loop through oseries names desc = "Get groundwater signatures" @@ -351,7 +351,7 @@ def get_signatures( s = self.conn.get_stresses(name) except Exception as e: if ignore_errors: - sign.loc[name, :] = np.nan + signatures_df.loc[name, :] = np.nan continue else: raise e @@ -371,9 +371,9 @@ def get_signatures( l.append(sign_val) else: raise e - sign.loc[name, signatures] = l + signatures_df.loc[name, signatures] = l - return sign + return signatures_df def get_tmin_tmax(self, libname, names=None, progressbar=False): """Get tmin and tmax for time series. From 92819574e19adbf1b05a6cf362c35135bc2b4a47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Tue, 24 Oct 2023 12:46:41 +0200 Subject: [PATCH 12/22] improve empty library - delete oseries_models when deleting models to avoid old models being stored in link dictionary --- pastastore/base.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pastastore/base.py b/pastastore/base.py index ffe98c8..5bd3886 100644 --- a/pastastore/base.py +++ b/pastastore/base.py @@ -934,13 +934,24 @@ def empty_library( ) if ui.lower() != "y": return - names = self._parse_names(None, libname) - for name in ( - tqdm(names, desc=f"Deleting items from {libname}") if progressbar else names - ): - self._del_item(libname, name) - self._clear_cache(libname) - print(f"Emptied library {libname} in {self.name}: " f"{self.__class__}") + + if libname == "models": + # also delete linked modelnames linked to oseries + libs = ["models", "oseries_models"] + else: + libs = [libname] + + # delete items and clear caches + for libname in libs: + names = self._parse_names(None, libname) + for name in ( + tqdm(names, desc=f"Deleting items from {libname}") + if progressbar + else names + ): + self._del_item(libname, name) + self._clear_cache(libname) + print(f"Emptied library {libname} in {self.name}: " f"{self.__class__}") def _iter_series(self, libname: str, names: Optional[List[str]] = None): """Internal method iterate over time series in library. From 6767be33a0b6638ebd4f47c6a6e1fa80f91fb7a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Tue, 24 Oct 2023 12:47:17 +0200 Subject: [PATCH 13/22] add get_extent to PastaStore --- pastastore/store.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pastastore/store.py b/pastastore/store.py index 15268d2..6f24b92 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -340,6 +340,23 @@ def get_tmin_tmax(self, libname, names=None, progressbar=False): tmintmax.loc[n, "tmax"] = s.last_valid_index() return tmintmax + def get_extent(self, libname, names=None): + names = self.conn._parse_names(names, libname=libname) + if libname in ["oseries", "stresses"]: + df = getattr(self, libname) + elif libname == "models": + df = self.oseries + else: + raise ValueError(f"Cannot get extent for library '{libname}'.") + + extent = [ + df.loc[names, "x"].min(), + df.loc[names, "x"].max(), + df.loc[names, "y"].min(), + df.loc[names, "y"].max(), + ] + return extent + def get_parameters( self, parameters: Optional[List[str]] = None, From 8d90e16a9536bd27b173c366c09c12ed93c40ab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Tue, 24 Oct 2023 12:58:22 +0200 Subject: [PATCH 14/22] black --- pastastore/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pastastore/base.py b/pastastore/base.py index 5bd3886..7434e9a 100644 --- a/pastastore/base.py +++ b/pastastore/base.py @@ -940,7 +940,7 @@ def empty_library( libs = ["models", "oseries_models"] else: libs = [libname] - + # delete items and clear caches for libname in libs: names = self._parse_names(None, libname) From a4ffe3e75cfe1c56ad2646bc90b1c58dbc8c039e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 26 Oct 2023 17:46:08 +0200 Subject: [PATCH 15/22] add buffer kwarg to extent - make it easy to get a slightly larger extent --- pastastore/store.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pastastore/store.py b/pastastore/store.py index 7e42dd6..c58f815 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -415,7 +415,7 @@ def get_tmin_tmax(self, libname, names=None, progressbar=False): tmintmax.loc[n, "tmax"] = s.last_valid_index() return tmintmax - def get_extent(self, libname, names=None): + def get_extent(self, libname, names=None, buffer=0.0): names = self.conn._parse_names(names, libname=libname) if libname in ["oseries", "stresses"]: df = getattr(self, libname) @@ -425,10 +425,10 @@ def get_extent(self, libname, names=None): raise ValueError(f"Cannot get extent for library '{libname}'.") extent = [ - df.loc[names, "x"].min(), - df.loc[names, "x"].max(), - df.loc[names, "y"].min(), - df.loc[names, "y"].max(), + df.loc[names, "x"].min() - buffer, + df.loc[names, "x"].max() + buffer, + df.loc[names, "y"].min() - buffer, + df.loc[names, "y"].max() + buffer, ] return extent From 7fc459fe36ea33017e0abb4635a1c5cbed8b53b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 26 Oct 2023 17:46:59 +0200 Subject: [PATCH 16/22] PastaStore.apply returns dataframe --- pastastore/store.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pastastore/store.py b/pastastore/store.py index c58f815..8de7bd8 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -1110,16 +1110,16 @@ def apply(self, libname, func, names=None, progressbar=True): Returns ------- - list - list of results of func + dict + dict of results of func, with names as keys and result as values """ names = self.conn._parse_names(names, libname) - result = [] + result = {} if libname not in ("oseries", "stresses", "models"): raise ValueError( "'libname' must be one of ['oseries', 'stresses', 'models']!" ) getter = getattr(self.conn, f"get_{libname}") for n in tqdm(names) if progressbar else names: - result.append(func(getter(n))) - return result + result[n] = func(getter(n)) + return pd.DataFrame.from_dict(result, orient="index") From d6d44e7aaa663c441577252cd1e32bac25e90ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 26 Oct 2023 17:47:03 +0200 Subject: [PATCH 17/22] version --- pastastore/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pastastore/version.py b/pastastore/version.py index 04980b4..e832ee3 100644 --- a/pastastore/version.py +++ b/pastastore/version.py @@ -4,4 +4,4 @@ PASTAS_VERSION = parse_version(ps.__version__) PASTAS_LEQ_022 = PASTAS_VERSION <= parse_version("0.22.0") -__version__ = "1.3.0b" +__version__ = "1.3.0" From 0ec3a442d8487333ec3bc3a947347a61f50afd88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 26 Oct 2023 17:55:34 +0200 Subject: [PATCH 18/22] apply should return dict, not DataFrame - e.g. when getting contributions from models --- pastastore/store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pastastore/store.py b/pastastore/store.py index 8de7bd8..96f8d6a 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -1111,7 +1111,7 @@ def apply(self, libname, func, names=None, progressbar=True): Returns ------- dict - dict of results of func, with names as keys and result as values + dict of results of func, with names as keys and results as values """ names = self.conn._parse_names(names, libname) result = {} @@ -1122,4 +1122,4 @@ def apply(self, libname, func, names=None, progressbar=True): getter = getattr(self.conn, f"get_{libname}") for n in tqdm(names) if progressbar else names: result[n] = func(getter(n)) - return pd.DataFrame.from_dict(result, orient="index") + return result From 3c434f66777ecf2963ed149cb81b730b1a21ffcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Wed, 1 Nov 2023 11:04:44 +0100 Subject: [PATCH 19/22] improve default Connectors - add default dictconnector to from_zip - ensure new connector instance is created for each new PastaStore. - improve documentation --- pastastore/store.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pastastore/store.py b/pastastore/store.py index 96f8d6a..b70c941 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -42,7 +42,7 @@ class PastaStore: def __init__( self, - connector: BaseConnector = DictConnector("pastas_db"), + connector: Optional[BaseConnector] = None, name: Optional[str] = None, ): """Initialize PastaStore for managing pastas time series and models. @@ -50,8 +50,9 @@ def __init__( Parameters ---------- connector : Connector object, optional - object that provides the connection to the database. Default is - DictConnector which does not store data on disk. + object that provides the connection to the database. Default is None, which + will create a DictConnector. This default Connector does not store data on + disk. name : str, optional name of the PastaStore, if not provided uses the Connector name """ @@ -59,6 +60,8 @@ def __init__( raise DeprecationWarning( "PastaStore expects the connector as the first argument since v1.1!" ) + if connector is None: + connector = DictConnector("pastas_db") self.conn = connector self.name = name if name is not None else self.conn.name self._register_connector_methods() @@ -934,7 +937,7 @@ def export_model_series_to_csv( def from_zip( cls, fname: str, - conn, + conn: Optional[BaseConnector] = None, storename: Optional[str] = None, progressbar: bool = True, ): @@ -944,8 +947,9 @@ def from_zip( ---------- fname : str pathname of zipfile - conn : Connector object - connector for storing loaded data + conn : Connector object, optional + connector for storing loaded data, default is None which creates a + DictConnector. This Connector does not store data on disk. storename : str, optional name of the PastaStore, by default None, which defaults to the name of the Connector. @@ -959,6 +963,9 @@ def from_zip( """ from zipfile import ZipFile + if conn is None: + conn = DictConnector("pastas_db") + with ZipFile(fname, "r") as archive: namelist = [ fi for fi in archive.namelist() if not fi.endswith("_meta.json") From 38e6a4997719cf70897dafa2def14e1cb40320c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Wed, 1 Nov 2023 11:41:43 +0100 Subject: [PATCH 20/22] codacy --- pastastore/store.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pastastore/store.py b/pastastore/store.py index b70c941..1433ec2 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -366,21 +366,21 @@ def get_signatures( raise e try: - l = ps.stats.signatures.summary(s.squeeze(), signatures) + i_signatures = ps.stats.signatures.summary(s.squeeze(), signatures) except Exception as e: if ignore_errors: - l = [] + i_signatures = [] for signature in signatures: try: sign_val = getattr(ps.stats.signatures, signature)( s.squeeze() ) - except: + except Exception as _: sign_val = np.nan - l.append(sign_val) + i_signatures.append(sign_val) else: raise e - signatures_df.loc[name, signatures] = l + signatures_df.loc[name, signatures] = i_signatures return signatures_df From a4a2bc4e8b7dc48460664be791525e8cb4d55e56 Mon Sep 17 00:00:00 2001 From: OnnoEbbens Date: Tue, 14 Nov 2023 14:14:53 +0100 Subject: [PATCH 21/22] fix for #108 --- pastastore/store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pastastore/store.py b/pastastore/store.py index 1433ec2..81c7c3e 100644 --- a/pastastore/store.py +++ b/pastastore/store.py @@ -973,7 +973,7 @@ def from_zip( for f in tqdm(namelist, desc="Reading zip") if progressbar else namelist: libname, fjson = os.path.split(f) if libname in ["stresses", "oseries"]: - s = pd.read_json(archive.open(f), orient="columns") + s = pd.read_json(archive.open(f), dtype=float, orient="columns") if not isinstance(s.index, pd.DatetimeIndex): s.index = pd.to_datetime(s.index, unit="ms") s = s.sort_index() From 07a0eb2b87407ad379c2545c71b381f2162ccc80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Mon, 27 Nov 2023 17:45:04 +0100 Subject: [PATCH 22/22] ruff --- docs/conf.py | 2 +- .../notebooks/ex02_pastastore_plots_and_maps.ipynb | 4 ---- .../notebooks/ex03_pastastore_yaml_interface.ipynb | 8 ++++---- pastastore/__init__.py | 1 + pastastore/plotting.py | 11 ++--------- tests/test_001_import.py | 2 +- tests/test_002_connectors.py | 8 ++++---- tests/test_003_pastastore.py | 13 ++++++++----- tests/test_004_yaml.py | 1 - 9 files changed, 21 insertions(+), 29 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1da1d1a..387af6b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ import sys sys.path.insert(0, os.path.abspath(".")) -from pastastore import __version__ +from pastastore import __version__ # noqa: E402 # -- Project information ----------------------------------------------------- diff --git a/examples/notebooks/ex02_pastastore_plots_and_maps.ipynb b/examples/notebooks/ex02_pastastore_plots_and_maps.ipynb index 4c2869c..71ce336 100644 --- a/examples/notebooks/ex02_pastastore_plots_and_maps.ipynb +++ b/examples/notebooks/ex02_pastastore_plots_and_maps.ipynb @@ -31,10 +31,6 @@ "metadata": {}, "outputs": [], "source": [ - "import sys\n", - "import os\n", - "\n", - "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "import pastastore as pst\n", "import pastas as ps\n", diff --git a/examples/notebooks/ex03_pastastore_yaml_interface.ipynb b/examples/notebooks/ex03_pastastore_yaml_interface.ipynb index 073f445..20cfd4b 100644 --- a/examples/notebooks/ex03_pastastore_yaml_interface.ipynb +++ b/examples/notebooks/ex03_pastastore_yaml_interface.ipynb @@ -163,7 +163,7 @@ "metadata": {}, "outputs": [], "source": [ - "from pastastore.datasets import example_pastastore" + "from pastastore.datasets import example_pastastore # noqa: E402" ] }, { @@ -364,7 +364,7 @@ ], "source": [ "ml.solve(report=False)\n", - "ml.plots.results();" + "ml.plots.results()" ] }, { @@ -802,7 +802,7 @@ ], "source": [ "ml.solve(report=False)\n", - "ml.plots.results();" + "ml.plots.results()" ] }, { @@ -910,7 +910,7 @@ ], "source": [ "ml.solve(report=False)\n", - "ml.plots.results();" + "ml.plots.results()" ] }, { diff --git a/pastastore/__init__.py b/pastastore/__init__.py index 57aad16..6f258d0 100644 --- a/pastastore/__init__.py +++ b/pastastore/__init__.py @@ -1,3 +1,4 @@ +# ruff: noqa: F401 from pastastore import connectors, util from pastastore.connectors import ( ArcticConnector, diff --git a/pastastore/plotting.py b/pastastore/plotting.py index 49669a3..834f19f 100644 --- a/pastastore/plotting.py +++ b/pastastore/plotting.py @@ -14,9 +14,6 @@ ax = pstore.maps.oseries() pstore.maps.add_background_map(ax) # for adding a background map """ - -from collections.abc import Iterable - import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -96,15 +93,11 @@ def _timeseries( if ax is None: if split: - fig, axes = plt.subplots(len(names), 1, sharex=True, figsize=figsize) + _, axes = plt.subplots(len(names), 1, sharex=True, figsize=figsize) else: - fig, axes = plt.subplots(1, 1, figsize=figsize) + _, axes = plt.subplots(1, 1, figsize=figsize) else: axes = ax - if isinstance(axes, Iterable): - fig = axes[0].figure - else: - fig = axes.figure tsdict = self.pstore.conn._get_series( libname, names, progressbar=progressbar, squeeze=False diff --git a/tests/test_001_import.py b/tests/test_001_import.py index f8d8456..6522e5a 100644 --- a/tests/test_001_import.py +++ b/tests/test_001_import.py @@ -4,4 +4,4 @@ def test_import(): with warnings.catch_warnings(): warnings.simplefilter(action="ignore", category=FutureWarning) - import pastastore + import pastastore # noqa: F401 diff --git a/tests/test_002_connectors.py b/tests/test_002_connectors.py index 8989218..f99d0b5 100644 --- a/tests/test_002_connectors.py +++ b/tests/test_002_connectors.py @@ -14,7 +14,7 @@ def test_get_library(conn): - olib = conn._get_library("oseries") + _ = conn._get_library("oseries") def test_add_get_series(request, conn): @@ -206,13 +206,13 @@ def test_add_stress(conn): @pytest.mark.dependency() def test_get_oseries(request, conn): depends(request, [f"test_add_oseries[{conn.type}]"]) - o = conn.get_oseries("oseries1") + _ = conn.get_oseries("oseries1") @pytest.mark.dependency() def test_get_oseries_and_metadata(request, conn): depends(request, [f"test_add_oseries[{conn.type}]"]) - o, m = conn.get_oseries("oseries1", return_metadata=True) + _ = conn.get_oseries("oseries1", return_metadata=True) @pytest.mark.dependency() @@ -225,7 +225,7 @@ def test_get_stress(request, conn): @pytest.mark.dependency() def test_get_stress_and_metadata(request, conn): depends(request, [f"test_add_stress[{conn.type}]"]) - s, m = conn.get_stresses("prec", return_metadata=True) + s, _ = conn.get_stresses("prec", return_metadata=True) s.name = "prec" diff --git a/tests/test_003_pastastore.py b/tests/test_003_pastastore.py index 0dd11c1..29feae3 100644 --- a/tests/test_003_pastastore.py +++ b/tests/test_003_pastastore.py @@ -39,7 +39,7 @@ def test_search(pstore): @pytest.mark.dependency() def test_create_model(pstore): - ml = pstore.create_model("oseries1") + _ = pstore.create_model("oseries1") @pytest.mark.dependency() @@ -139,7 +139,7 @@ def test_get_model(request, pstore): f"test_store_model_missing_series[{pstore.type}]", ], ) - ml = pstore.conn.get_models("oseries1") + _ = pstore.conn.get_models("oseries1") @pytest.mark.dependency() @@ -158,7 +158,7 @@ def test_del_model(request, pstore): @pytest.mark.dependency() def test_create_models(pstore): - mls = pstore.create_models_bulk( + _ = pstore.create_models_bulk( ["oseries1", "oseries2"], store=True, progressbar=False ) _ = pstore.conn.models @@ -188,7 +188,7 @@ def test_iter_models(request, pstore): @pytest.mark.dependency() def test_solve_models_and_get_stats(request, pstore): depends(request, [f"test_create_models[{pstore.type}]"]) - mls = pstore.solve_models( + _ = pstore.solve_models( ignore_solve_errors=False, progressbar=False, store_result=True ) stats = pstore.get_statistics(["evp", "aic"], progressbar=False) @@ -198,7 +198,10 @@ def test_solve_models_and_get_stats(request, pstore): @pytest.mark.dependency() def test_apply(request, pstore): depends(request, [f"test_solve_models_and_get_stats[{pstore.type}]"]) - func = lambda ml: ml.parameters.loc["recharge_A", "optimal"] + + def func(ml): + return ml.parameters.loc["recharge_A", "optimal"] + result = pstore.apply("models", func) assert len(result) == 2 diff --git a/tests/test_004_yaml.py b/tests/test_004_yaml.py index c279912..fb165c0 100644 --- a/tests/test_004_yaml.py +++ b/tests/test_004_yaml.py @@ -6,7 +6,6 @@ from pytest_dependency import depends import pastastore as pst -from pastastore.version import PASTAS_LEQ_022 @contextmanager