diff --git a/.github/workflows/test-conda.yaml b/.github/workflows/test-conda.yaml index 0211022..78995c9 100644 --- a/.github/workflows/test-conda.yaml +++ b/.github/workflows/test-conda.yaml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-12, windows-latest] python: ["3.9", "3.10"] steps: @@ -29,7 +29,7 @@ jobs: - name: Setup Environment shell: bash run: | - conda create --name test python=${{ matrix.python }} pytest numpy gdal=3.* scipy pytz dask-core toolz geopandas pyproj>=2 + conda create --name test python=${{ matrix.python }} pytest numpy=1.* gdal=3.* scipy pytz dask-core toolz "pandas<2.2" geopandas=0.* "pyproj>=2" source activate test python -V conda info diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 82f6222..2576b3f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,8 +37,8 @@ jobs: pins: "pygdal==3.4.1.* scipy==1.9.* dask[delayed]==2021.7.* pandas==1.4.* geopandas==0.11.*" - os: ubuntu-22.04 python: "3.11" - display_name: "latest" - pins: "pygdal==3.4.1.*" + numpy: "==1.*" + pins: "pygdal==3.4.1.* geopandas==0.* pandas==2.1.*" steps: - uses: actions/checkout@v2 diff --git a/CHANGES.rst b/CHANGES.rst index 12c0cde..87fad4d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,10 +1,13 @@ Changelog of dask-geomodeling =================================================== -2.4.5 (unreleased) +2.5.0 (unreleased) ------------------ -- Nothing changed yet. +- Added mandatory `temporal` attribute for RasterBlock. + +- Added version constraints showing incompatibility with geopandas 1.*, + pandas 2.2, and numpy 2. 2.4.4 (2024-01-17) diff --git a/dask_geomodeling/raster/base.py b/dask_geomodeling/raster/base.py index 40b1f0f..e54cdef 100644 --- a/dask_geomodeling/raster/base.py +++ b/dask_geomodeling/raster/base.py @@ -12,14 +12,15 @@ class RasterBlock(Block): All RasterBlocks must be derived from this base class and must implement the following attributes: - - ``period``: a tuple of datetimes - - ``timedelta``: a datetime.timedelta (or None if nonequidistant) + - ``period``: a tuple of datetimes (or None if empty) + - ``timedelta``: a datetime.timedelta (or None if nonequidistant / nontemporal) - ``extent``: a tuple ``(x1, y1, x2, y2)`` - ``dtype``: a numpy dtype object - ``fillvalue``: a number - ``geometry``: OGR Geometry - ``projection``: WKT string - ``geo_transform``: a tuple of 6 numbers + - ``temporal``: a boolean indicating whether the raster is temporal These attributes are ``None`` if the raster is empty. @@ -178,6 +179,10 @@ def period(self): def timedelta(self): return self.store.timedelta + @property + def temporal(self): + return self.store.temporal + @property def dtype(self): return self.store.dtype diff --git a/dask_geomodeling/raster/combine.py b/dask_geomodeling/raster/combine.py index 921b679..5772f5a 100644 --- a/dask_geomodeling/raster/combine.py +++ b/dask_geomodeling/raster/combine.py @@ -62,6 +62,10 @@ def get_aligned_timedelta(sources): def timedelta(self): """ The period between timesteps in case of equidistant time. """ return self.get_aligned_timedelta(self.args) + + @property + def temporal(self): + return any(x.temporal for x in self.args) @property def period(self): diff --git a/dask_geomodeling/raster/elemwise.py b/dask_geomodeling/raster/elemwise.py index 049f2f8..76c1cf1 100644 --- a/dask_geomodeling/raster/elemwise.py +++ b/dask_geomodeling/raster/elemwise.py @@ -45,9 +45,16 @@ class BaseElementwise(RasterBlock): def __init__(self, *args): super(BaseElementwise, self).__init__(*args) - # check the timedelta so that an error is raised if incompatible + # check the temporal and timedelta attributes of the sources if len(self._sources) > 1: - self.timedelta # NOQA + temporal, delta = self._sources[0].temporal, self._sources[0].timedelta + + if any(source.temporal != temporal for source in self._sources[1:]): + raise ValueError("Temporal properties of input rasters do not match.") + + if temporal and (delta is not None): + if not all(source.timedelta in (None, delta) for source in self._sources[1:]): + raise ValueError("Time resolutions of input rasters are not equal.") @property def _sources(self): @@ -75,20 +82,17 @@ def timedelta(self): """ The period between timesteps in case of equidistant time. """ if len(self._sources) == 1: return self._sources[0].timedelta - timedeltas = [s.timedelta for s in self._sources] - if any(timedelta is None for timedelta in timedeltas): - return None # None precedes - - # multiple timedeltas: assert that they are equal - if not timedeltas[1:] == timedeltas[:-1]: - raise ValueError( - "Time resolutions of input rasters are not " - "equal ({}).".format(timedeltas) - ) + if any(x is None for x in timedeltas): + return None else: return timedeltas[0] + @property + def temporal(self): + """Temporal property of sources is enforced to be equal in the init""" + return self._sources[0].temporal + @property def period(self): """ Return period datetime tuple. """ @@ -97,7 +101,7 @@ def period(self): periods = [s.period for s in self._sources] if any(period is None for period in periods): - return None # None precedes + return None # None (empty raster) precedes # multiple periods: return the overlapping period start = max([p[0] for p in periods]) diff --git a/dask_geomodeling/raster/misc.py b/dask_geomodeling/raster/misc.py index 5a5098b..f30e92d 100644 --- a/dask_geomodeling/raster/misc.py +++ b/dask_geomodeling/raster/misc.py @@ -51,8 +51,17 @@ class Clip(BaseSingle): def __init__(self, store, source): if not isinstance(source, RasterBlock): raise TypeError("'{}' object is not allowed".format(type(store))) - # timedeltas are required to be equal - if store.timedelta != source.timedelta: + if store.temporal and not source.temporal: + raise ValueError( + "The values raster is temporal while the clipping mask is not. " + "Consider using Snap." + ) + if not store.temporal and source.temporal: + raise ValueError( + "The clipping mask is temporal while the values raster is not. " + "Consider using Snap." + ) + if store.temporal and (store.timedelta != source.timedelta): raise ValueError( "Time resolution of the clipping mask does not match that of " "the values raster. Consider using Snap." @@ -572,6 +581,10 @@ def extent(self): @property def timedelta(self): return None + + @property + def temporal(self): + return False @property def geometry(self): @@ -743,6 +756,10 @@ def extent(self): @property def timedelta(self): return None + + @property + def temporal(self): + return False @property def geometry(self): diff --git a/dask_geomodeling/raster/sources.py b/dask_geomodeling/raster/sources.py index d4a19ef..225287e 100644 --- a/dask_geomodeling/raster/sources.py +++ b/dask_geomodeling/raster/sources.py @@ -182,9 +182,14 @@ def period(self): @property def timedelta(self): - if len(self) <= 1: + if self.time_delta is None: return None - return timedelta(milliseconds=self.time_delta) + else: + return timedelta(milliseconds=self.time_delta) + + @property + def temporal(self): + return self.time_delta is not None def get_sources_and_requests(self, **request): mode = request["mode"] @@ -404,6 +409,10 @@ def timedelta(self): if len(self) <= 1: return None return timedelta(milliseconds=self.time_delta) + + @property + def temporal(self): + return len(self) > 1 def get_sources_and_requests(self, **request): mode = request["mode"] diff --git a/dask_geomodeling/raster/temporal.py b/dask_geomodeling/raster/temporal.py index 1e54893..abbfb93 100644 --- a/dask_geomodeling/raster/temporal.py +++ b/dask_geomodeling/raster/temporal.py @@ -77,6 +77,10 @@ def period(self): @property def timedelta(self): return self.index.timedelta + + @property + def temporal(self): + return self.index.temporal @property def extent(self): @@ -553,6 +557,10 @@ def timedelta(self): except AttributeError: return # e.g. Month is non-equidistant + @property + def temporal(self): + return self.frequency is not None + @property def dtype(self): return dtype_for_statistic(self.source.dtype, self.statistic) diff --git a/dask_geomodeling/tests/conftest.py b/dask_geomodeling/tests/conftest.py index 9eea742..4f7c695 100644 --- a/dask_geomodeling/tests/conftest.py +++ b/dask_geomodeling/tests/conftest.py @@ -38,6 +38,22 @@ def empty_source(): ) + +@pytest.fixture(scope="session") +def empty_temporal_source(): + time_first = datetime(2000, 1, 1) + time_delta = timedelta(hours=1) + yield MemorySource( + data=np.empty((0, 0, 0), dtype=np.uint8), + no_data_value=255, + projection="EPSG:28992", + pixel_size=0.5, + pixel_origin=(135000, 456000), + time_first=time_first, + time_delta=time_delta, + + ) + @pytest.fixture(scope="session") def nodata_source(): time_first = datetime(2000, 1, 1) diff --git a/dask_geomodeling/tests/factories.py b/dask_geomodeling/tests/factories.py index 548b477..b3b9d4f 100644 --- a/dask_geomodeling/tests/factories.py +++ b/dask_geomodeling/tests/factories.py @@ -35,13 +35,15 @@ class MockRaster(RasterBlock): """ def __init__( - self, origin=None, timedelta=None, bands=None, value=1, projection="EPSG:3857" + self, origin=None, timedelta=None, bands=None, value=1, projection="EPSG:3857", temporal=None ): self.origin = origin self._timedelta = timedelta self.bands = bands self.value = value - super(MockRaster, self).__init__(origin, timedelta, bands, value, projection) + if temporal is None: + temporal = timedelta is not None + super(MockRaster, self).__init__(origin, timedelta, bands, value, projection, temporal) @property def dtype(self): @@ -53,13 +55,17 @@ def dtype(self): @property def fillvalue(self): return get_dtype_max(self.dtype) + + @property + def temporal(self): + return self.args[5] def get_sources_and_requests(self, **request): return [(self.args, None), (request, None)] @staticmethod def process(args, request): - origin, timedelta, bands, value, src_projection = args + origin, timedelta, bands, value, src_projection, temporal = args if origin is None or timedelta is None or bands is None: return td_seconds = timedelta.total_seconds() @@ -155,6 +161,10 @@ def period(self): @property def timedelta(self): return self._timedelta + + @property + def temporal(self): + return self.args[5] @property def extent(self): diff --git a/dask_geomodeling/tests/test_raster.py b/dask_geomodeling/tests/test_raster.py index 44f1d1a..a35f6fe 100644 --- a/dask_geomodeling/tests/test_raster.py +++ b/dask_geomodeling/tests/test_raster.py @@ -45,6 +45,7 @@ def test_attrs(self): "geometry", "projection", "geo_transform", + "temporal" ): if not hasattr(klass, attr): print(name, attr) @@ -80,15 +81,31 @@ def test_propagate_timedelta(self): elemwise = self.klass(*args) self.assertEqual(elemwise.timedelta, storage1.timedelta) - def test_propagate_none_timedelta(self): + def test_propagate_nonequidistant_time(self): storage1 = MockRaster(timedelta=Timedelta(hours=1)) - storage2 = MockRaster(timedelta=None) + storage2 = MockRaster(timedelta=None, temporal=True) # None timedelta precedes for args in [(storage1, storage2), (storage2, storage1)]: elemwise = self.klass(*args) self.assertIsNone(elemwise.timedelta) + def test_propagate_temporal(self): + storage1 = MockRaster(timedelta=Timedelta(hours=1)) + storage2 = MockRaster(timedelta=None) + + # result is temporal if all inputs are temporal + elemwise = self.klass(storage1, storage1) + self.assertTrue(elemwise.temporal) + + # result is non-temporal if all inputs are non-temporal + elemwise = self.klass(storage2, storage2) + self.assertFalse(elemwise.temporal) + + # can't mix the two + self.assertRaises(ValueError, self.klass, storage1, storage2) + self.assertRaises(ValueError, self.klass, storage2, storage1) + def test_propagate_period(self): storage1 = MockRaster( origin=Datetime(2018, 4, 1), timedelta=Timedelta(hours=1), bands=6 @@ -118,7 +135,7 @@ def test_propagate_period(self): self.assertIsNone(elemwise.period) def test_propagate_none_period(self): - storage1 = MockRaster(origin=None) + storage1 = MockRaster(origin=None, temporal=True) storage2 = MockRaster( origin=Datetime(2018, 4, 1), timedelta=Timedelta(hours=1), bands=6 ) @@ -667,6 +684,20 @@ def test_propagate_timedelta(self): combined = self.klass(storage1, storage5) self.assertIsNone(combined.timedelta) + def test_propagate_temporal(self): + # if any input is temporal, the result is temporal + storage1 = MockRaster( + timedelta=Timedelta(hours=1) + ) + storage2 = MockRaster( + timedelta=None + ) + + self.assertTrue(self.klass(storage1, storage2).temporal) + self.assertTrue(self.klass(storage2, storage1).temporal) + self.assertTrue(self.klass(storage1, storage1).temporal) + self.assertFalse(self.klass(storage2, storage2).temporal) + def test_propagate_period(self): storage1 = MockRaster( origin=Datetime(2018, 4, 1), timedelta=Timedelta(hours=1), bands=6 @@ -1038,6 +1069,7 @@ def test_snap_attributes(self): self.assertEqual(self.view.period, self.index.period) self.assertEqual(self.view.timedelta, self.index.timedelta) self.assertEqual(len(self.view), len(self.index)) + self.assertEqual(self.view.temporal, self.index.temporal) def test_snap_empty_store_or_index(self): view = self.klass(self.raster, self.empty) @@ -1194,6 +1226,7 @@ def test_base_view(self): self.assertEqual(original.extent, view.extent) self.assertEqual(original.period, view.period) self.assertEqual(original.timedelta, view.timedelta) + self.assertEqual(original.temporal, view.temporal) def test_shift(self): # store and view @@ -1529,6 +1562,9 @@ def setUp(self): properties = [{"id": x, "value": x / 3} for x in (51, 212, 512)] self.geometry_source = MockGeometry(squares, properties) self.view = raster.Rasterize(self.geometry_source, "id") + + def test_attrs(self): + self.assertFalse(self.view.temporal) def test_vals_request(self): data = self.view.get_data(**self.vals_request) diff --git a/dask_geomodeling/tests/test_raster_elemwise.py b/dask_geomodeling/tests/test_raster_elemwise.py new file mode 100644 index 0000000..2a31d2a --- /dev/null +++ b/dask_geomodeling/tests/test_raster_elemwise.py @@ -0,0 +1,44 @@ +from dask_geomodeling.raster.elemwise import BaseElementwise +from dask_geomodeling.tests.factories import MockRaster +from datetime import datetime, timedelta +import pytest + + + + +@pytest.mark.parametrize("inverse", [False, True]) +@pytest.mark.parametrize("temporal1,delta1,temporal2,delta2,ok", + [ + # nontemporal - nontemporal + (False, None, False, None, True), # old: True + (False, timedelta(minutes=5), False, timedelta(minutes=5), True), # old: True + (False, None, False, timedelta(minutes=5), True), # old: True + + # nontemporal - temporal + # note that before "None" delta meant only "temporal nonequidistant" + # so we can safely have new behaviour for (False, None) cases + (False, None, True, None, False), # old: True + (False, None, True, timedelta(hours=1), False), # old: True + (False, timedelta(minutes=5), True, None, False), # old: True + (False, timedelta(minutes=5), True, timedelta(hours=1), False), # old: False + + # temporal - temporal + (True, timedelta(hours=1), True, timedelta(hours=1), True), # old: True + (True, timedelta(hours=1), True, timedelta(hours=2), False), # old: False + (True, timedelta(hours=1), True, None, True), # old: True + (True, None, True, None, True), # old: True + ] +) +def test_raster_elemwise_init_ok(delta1,temporal1,delta2,temporal2,inverse,ok): + raster1 = MockRaster(origin=datetime(2000, 1, 1), timedelta=delta1, temporal=temporal1) + raster2 = MockRaster(origin=datetime(2000, 1, 1), timedelta=delta2, temporal=temporal2) + + if inverse: + raster1, raster2 = raster2, raster1 + + if ok: + BaseElementwise(raster1, raster2) + else: + with pytest.raises(ValueError): + BaseElementwise(raster1, raster2) + diff --git a/dask_geomodeling/tests/test_raster_misc.py b/dask_geomodeling/tests/test_raster_misc.py index ae730ef..607b91b 100644 --- a/dask_geomodeling/tests/test_raster_misc.py +++ b/dask_geomodeling/tests/test_raster_misc.py @@ -8,23 +8,27 @@ from dask_geomodeling import raster from dask_geomodeling.utils import shapely_transform, get_sr from dask_geomodeling.raster.sources import MemorySource +from dask_geomodeling.tests.factories import ( + MockRaster, +) + -def test_clip_attrs_store_empty(source, empty_source): +def test_clip_attrs_store_empty(source, empty_temporal_source): # clip should propagate the (empty) extent of the store - clip = raster.Clip(empty_source, raster.Snap(source, empty_source)) + clip = raster.Clip(empty_temporal_source, source) assert clip.extent is None assert clip.geometry is None -def test_clip_attrs_mask_empty(source, empty_source): +def test_clip_attrs_mask_empty(source, empty_temporal_source): # clip should propagate the (empty) extent of the clipping mask - clip = raster.Clip(source, raster.Snap(empty_source, source)) + clip = raster.Clip(source, empty_temporal_source) assert clip.extent is None assert clip.geometry is None -def test_clip_attrs_intersects(source, empty_source): +def test_clip_attrs_intersects(source): # create a raster in that only partially overlaps the store clipping_mask = MemorySource( data=source.data, @@ -47,7 +51,7 @@ def test_clip_attrs_intersects(source, empty_source): assert clip.geometry.GetEnvelope() == expected_geometry.GetEnvelope() -def test_clip_attrs_with_reprojection(source, empty_source): +def test_clip_attrs_with_reprojection(source): # create a raster in WGS84 that contains the store clipping_mask = MemorySource( data=source.data, @@ -84,22 +88,22 @@ def test_clip_matching_timedelta(source): assert clip.timedelta == source.timedelta -def test_clip_unequal_timedelta(source, empty_source): - # clip checks for matching timedeltas; test that here +def test_clip_unequal_temporal(source, empty_source): + # clip checks for matching "temporal" attribute; test that here # NB: note that `source` is temporal and `empty_source` is not - with pytest.raises(ValueError, match=".*resolution of the clipping.*"): + with pytest.raises(ValueError, match=".*Consider using Snap.*"): raster.Clip(source, empty_source) - with pytest.raises(ValueError, match=".*resolution of the clipping.*"): + with pytest.raises(ValueError, match=".*Consider using Snap.*"): raster.Clip(empty_source, source) -def test_clip_empty_source(source, empty_source, vals_request): - clip = raster.Clip(empty_source, raster.Snap(source, empty_source)) +def test_clip_empty_source(source, empty_temporal_source, vals_request): + clip = raster.Clip(empty_temporal_source, source) assert clip.get_data(**vals_request) is None -def test_clip_with_empty_mask(source, empty_source, vals_request): - clip = raster.Clip(source, raster.Snap(empty_source, source)) +def test_clip_with_empty_mask(source, empty_temporal_source, vals_request): + clip = raster.Clip(source, empty_temporal_source) assert clip.get_data(**vals_request) is None @@ -290,3 +294,4 @@ def test_rasterize_wkt_attrs(): ) assert view.timedelta is None assert view.period == (datetime(1970, 1, 1), datetime(1970, 1, 1)) + assert view.temporal is False diff --git a/dask_geomodeling/tests/test_raster_reduction.py b/dask_geomodeling/tests/test_raster_reduction.py index f5dc11e..0b1dfd9 100644 --- a/dask_geomodeling/tests/test_raster_reduction.py +++ b/dask_geomodeling/tests/test_raster_reduction.py @@ -111,8 +111,8 @@ def test_max_with_nodata(source, nodata_source, vals_request): assert data["values"][:, 0, 0].tolist() == [1, 7, data["no_data_value"]] -def test_max_with_empty(source, empty_source, vals_request): - block = raster.Max(source, empty_source) +def test_max_with_empty(source, empty_temporal_source, vals_request): + block = raster.Max(source, empty_temporal_source) data = block.get_data(**vals_request) assert data is None diff --git a/dask_geomodeling/tests/test_raster_sources.py b/dask_geomodeling/tests/test_raster_sources.py index 502e34b..ee8b1b8 100644 --- a/dask_geomodeling/tests/test_raster_sources.py +++ b/dask_geomodeling/tests/test_raster_sources.py @@ -23,6 +23,9 @@ def test_period(self): def test_timedelta(self): self.assertEqual(timedelta(days=1), self.source.timedelta) + def test_temporal(self): + self.assertTrue(self.source.temporal) + def test_len(self): self.assertEqual(2, len(self.source)) diff --git a/dask_geomodeling/tests/test_raster_temporal.py b/dask_geomodeling/tests/test_raster_temporal.py index b4c8a83..1504fac 100644 --- a/dask_geomodeling/tests/test_raster_temporal.py +++ b/dask_geomodeling/tests/test_raster_temporal.py @@ -200,6 +200,11 @@ def test_timedelta(self): # months are nonequidistant self.assertIsNone(self.klass(self.raster, "M").timedelta) + def test_temporal(self): + self.assertTrue(self.klass(self.raster, "D").temporal) # equidistant + self.assertTrue(self.klass(self.raster, "M").temporal) # non-equidistant + self.assertFalse(self.klass(self.raster, None).temporal) # non-temporal + def test_get_data_time_request(self): self.view = self.klass(self.raster, "H", closed="left", label="right") self.request["mode"] = "time" diff --git a/setup.py b/setup.py index bdd4090..cf3d02c 100644 --- a/setup.py +++ b/setup.py @@ -8,11 +8,12 @@ install_requires = ( [ "dask[delayed]>=0.20", - "pandas>=0.23", - "geopandas>=0.7", + "pandas>=0.23,<=2.2", + "geopandas>=0.7,<1", "pytz", - "numpy>=1.14", + "numpy>=1.14,<2", "scipy>=1.1", + "fiona" ], )