From 714e81b4a2d4ad3348e9c4f48da8c496b169aec2 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Thu, 16 Jan 2025 13:05:13 -0700 Subject: [PATCH 01/23] switching to use flake8 from github instead of gitlab and updating repo versions --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8aae92a47..a9208fcab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v5.0.0 hooks: - id: check-json - id: check-yaml @@ -13,11 +13,11 @@ repos: - id: requirements-txt-fixer - id: pretty-format-json args: [--autofix] -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 +- repo: https://github.com/PyCQA/flake8 + rev: 7.1.1 hooks: - id: flake8 - repo: https://github.com/PyCQA/pylint - rev: pylint-2.6.0 + rev: v3.3.3 hooks: - id: pylint From 31f032b2f1e578fd5ee4f399c9d9b6f6a6b33451 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Thu, 16 Jan 2025 13:20:09 -0700 Subject: [PATCH 02/23] updating max arguments to 25, removing deprecated options from disable clause --- .pylintrc | 63 +------------------------------------------------------ 1 file changed, 1 insertion(+), 62 deletions(-) diff --git a/.pylintrc b/.pylintrc index 3b3916798..c74e46561 100644 --- a/.pylintrc +++ b/.pylintrc @@ -58,12 +58,6 @@ disable= unspecified-encoding, consider-using-f-string, # Defaults - print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - import-star-module-level, raw-checker-failed, bad-inline-option, locally-disabled, @@ -71,60 +65,6 @@ disable= suppressed-message, useless-suppression, deprecated-pragma, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, # Custom protected-access, fixme, @@ -145,7 +85,6 @@ disable= too-many-nested-blocks, invalid-name, import-error, - bad-continuation, try-except-raise, no-else-raise, no-else-return, @@ -491,7 +430,7 @@ valid-metaclass-classmethod-first-arg=mcs [DESIGN] # Maximum number of arguments for function / method -max-args=5 +max-args=25 # Maximum number of attributes for a class (see R0902). max-attributes=7 From 4cc2795ae44e989ca8b75ff6e7f8a3b38a3af5cb Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Thu, 16 Jan 2025 13:22:05 -0700 Subject: [PATCH 03/23] reverting increase of max-args and adding new setting for max-positional-args (25) --- .pylintrc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index c74e46561..0f9b47b4a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -430,7 +430,10 @@ valid-metaclass-classmethod-first-arg=mcs [DESIGN] # Maximum number of arguments for function / method -max-args=25 +max-args=5 + +# Maximum number of position arguments for function / method +max-positional-arguments=25 # Maximum number of attributes for a class (see R0902). max-attributes=7 From fd7e48886a447e72017b299c7d5d007243cfab92 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Thu, 16 Jan 2025 13:22:18 -0700 Subject: [PATCH 04/23] adding zone masking to supply curve points --- reV/supply_curve/points.py | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 5829d9874..975e37cbc 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -221,6 +221,7 @@ def __init__( excl_area=None, exclusion_shape=None, close=True, + zone_mask=None, ): """ Parameters @@ -254,6 +255,10 @@ def __init__( will speed things up considerably. close : bool Flag to close object file handlers on exit. + zone_mask : np.ndarray | None, optional + 2D array defining zone within the supply curve to be evaluated, + where 1 is included and 0 is excluded. The shape of this will be + checked against the input resolution. """ self._excl_dict = excl_dict @@ -282,6 +287,9 @@ def __init__( assert inclusion_mask.size == len(self._gids), msg self._incl_mask = inclusion_mask.copy() + self._zone_mask = zone_mask + self._check_zone_mask() + self._centroid = None self._excl_area = excl_area self._check_excl() @@ -454,6 +462,24 @@ def n_gids(self): return n_gids + @property + def zone_mask(self): + """ + Get the 2D zone mask, where 1 is included and 0 is excluded. + + Returns + ------- + np.ndarray + """ + + if self._zone_mask is None: + return None + else: + out_of_extent = self._gids.reshape(self._zone_mask.shape) == -1 + self._zone_mask[out_of_extent] = 0.0 + + return self._zone_mask + @property def include_mask(self): """Get the 2D inclusion mask (normalized with expected range: [0, 1] @@ -481,6 +507,10 @@ def include_mask(self): logger.warning(w) warn(w) + if self.zone_mask is not None: + out_of_zone = self.zone_mask == 0 + self._incl_mask[out_of_zone] = 0.0 + return self._incl_mask @property @@ -548,6 +578,31 @@ def _check_excl(self): ) raise EmptySupplyCurvePointError(msg) + def _check_zone_mask(self): + """ + Check that the zone mask is the correct size and shape, and that it + contains only values of 0 and 1. + """ + if self._zone_mask is not None: + msg = ( + "Bad zone mask input shape of {} with stated " + "resolution of {}".format( + self._zone_mask.shape, self._resolution + ) + ) + assert len(self._zone_mask.shape) == 2, msg + assert self._zone_mask.shape[0] <= self._resolution, msg + assert self._zone_mask.shape[1] <= self._resolution, msg + assert self._zone_mask.size == len(self._gids), msg + + if not np.isin(self._zone_mask, [0, 1]).all(): + msg = ( + "zone_mask includes unexpected values. All values must be " + "in the domain: [0, 1]" + ) + logger.error(msg) + raise ValueError(msg) + def exclusion_weighted_mean(self, arr, drop_nan=True): """ Calc the exclusions-weighted mean value of an array of resource data. @@ -955,6 +1010,7 @@ def __init__( close=True, gen_index=None, apply_exclusions=True, + zone_mask=None, ): """ Parameters @@ -997,6 +1053,10 @@ def __init__( apply_exclusions : bool Flag to apply exclusions to the resource / generation gid's on initialization. + zone_mask : np.ndarray | None, optional + 2D array defining zone within the supply curve to be evaluated, + where 1 is included and 0 is excluded. The shape of this will be + checked against the input resolution. """ super().__init__( gid, @@ -1008,6 +1068,7 @@ def __init__( excl_area=excl_area, exclusion_shape=exclusion_shape, close=close, + zone_mask=zone_mask, ) self._h5_fpath, self._h5 = self._parse_h5_file(agg_h5) @@ -1435,6 +1496,7 @@ def __init__( friction_layer=None, recalc_lcoe=True, apply_exclusions=True, + zone_mask=None, ): """ Parameters @@ -1510,6 +1572,10 @@ def __init__( apply_exclusions : bool Flag to apply exclusions to the resource / generation gid's on initialization. + zone_mask : np.ndarray | None, optional + 2D array defining zone within the supply curve to be summarized, + where 1 is included and 0 is excluded. The shape of this will be + checked against the input resolution. """ self._res_class_dset = res_class_dset @@ -1540,6 +1606,7 @@ def __init__( exclusion_shape=exclusion_shape, close=close, apply_exclusions=False, + zone_mask=zone_mask, ) self._res_gid_set = None @@ -2423,6 +2490,7 @@ def summarize( data_layers=None, cap_cost_scale=None, recalc_lcoe=True, + zone_mask=None, ): """Get a summary dictionary of a single supply curve point. @@ -2504,6 +2572,11 @@ def summarize( datasets to be aggregated in the gen input: system_capacity, fixed_charge_rate, capital_cost, fixed_operating_cost, and variable_operating_cost. + zone_mask : np.ndarray | None, optional + 2D array defining zone within the supply curve to be summarized, + where 1 is included and 0 is excluded. The shape of this will be + checked against the input resolution. If not specified, no zone + mask will be applied. Returns ------- @@ -2525,6 +2598,7 @@ def summarize( "close": close, "friction_layer": friction_layer, "recalc_lcoe": recalc_lcoe, + "zone_mask": zone_mask, } with cls( From 063ab4ffcefc3a0f8bd5c7d581aa773077533790 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Thu, 16 Jan 2025 13:23:32 -0700 Subject: [PATCH 05/23] adding functionality to load zones array for a specified gid --- reV/supply_curve/aggregation.py | 47 +++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index ac405ba9c..1879dc6e4 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -438,6 +438,53 @@ def _get_gid_inclusion_mask( return gid_inclusions + @staticmethod + def _get_gid_zones(zones_fpath, zones_dset, gid, slice_lookup): + """ + Get zones 2D array for desired gid. + + Parameters + ---------- + zones_fpath : str | None, optional + Filepath to HDF5 file containing `zones_dset`. If not specified, + output of function will be an array containing all values equal to + 1. + zones_dset : str | None, optional + Dataset name in the `zones_fpath` file containing the zones to be + loaded. If not specified, output of function will be an array + containing all values equal to 1. + gid : int + sc_point_gid value, used to extract the applicable subset of zones. + slice_lookup : dict + Mapping of sc_point_gids to exclusion/inclusion row and column + slices + + Returns + ------- + zones : ndarray | None + 2D array of zones for desired gid. + """ + + row_slice, col_slice = slice_lookup[gid] + if zones_fpath is not None and zones_dset is not None: + with ExclusionLayers(zones_fpath) as fh: + if zones_dset not in fh: + msg = ( + f"Could not find zones_dset {zones_dset} in " + f"zones_fpath {zones_fpath}." + ) + logger.error(msg) + raise FileInputError(msg) + zones = fh[zones_dset, row_slice, col_slice] + else: + shape = ( + row_slice.stop - row_slice.start, + col_slice.stop - col_slice.start + ) + zones = np.ones(shape, dtype="uint32") + + return zones + @staticmethod def _parse_gen_index(gen_fpath): """Parse gen outputs for an array of generation gids corresponding to From d1178e733beb6fda88e82742d0bd74dda9791c95 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Thu, 16 Jan 2025 13:25:59 -0700 Subject: [PATCH 06/23] bumping max-positional-arguments to 30 --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 0f9b47b4a..a936f8422 100644 --- a/.pylintrc +++ b/.pylintrc @@ -433,7 +433,7 @@ valid-metaclass-classmethod-first-arg=mcs max-args=5 # Maximum number of position arguments for function / method -max-positional-arguments=25 +max-positional-arguments=30 # Maximum number of attributes for a class (see R0902). max-attributes=7 From d7b6123be417349209a722e5dff73972e9c986a8 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Thu, 16 Jan 2025 13:26:11 -0700 Subject: [PATCH 07/23] adding input arguments and logic for aggregating within zones of gids --- reV/supply_curve/sc_aggregation.py | 140 +++++++++++++++++++++-------- 1 file changed, 102 insertions(+), 38 deletions(-) diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index 40c3bf7c3..b71ec98f9 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -252,7 +252,8 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, res_class_bins=None, cf_dset='cf_mean-means', lcoe_dset='lcoe_fcr-means', h5_dsets=None, data_layers=None, power_density=None, friction_fpath=None, friction_dset=None, - cap_cost_scale=None, recalc_lcoe=True): + cap_cost_scale=None, recalc_lcoe=True, zones_fpath=None, + zones_dset=None): r"""ReV supply curve points aggregation framework. ``reV`` supply curve aggregation combines a high-resolution @@ -297,7 +298,6 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, exclusion HDF5 file is a blocking operation, so you may only run a single ``reV`` aggregation step at a time this way. - econ_fpath : str, optional Filepath to HDF5 file with ``reV`` econ output results containing an `lcoe_dset` dataset. If ``None``, `lcoe_dset` @@ -544,6 +544,22 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, generation HDF5 output, or if `recalc_lcoe` is set to ``False``, the mean LCOE will be computed from the data stored under the `lcoe_dset` instead. By default, ``True``. + zones_fpath : str, optional + Filepath to HDF5 file containing `zones_dset`. If both + `zones_fpath` and `zones_dset` are specified, supply curve + aggregation will be applied separately for each zone within each + supply curve site. This file should match the format of a typical + exclusions HDF5 file. If specified without zones_dset, a warning + will be logged. + zones_dset: str, optoinal + Dataset name in the `zones_fpath` file containing the zones to be + applied. This data layer should consist of unique integer values + for each zone, and should be consistent in shape with datasets in + `excl_fpath`. Values of zero will be treated as no data /ignored. + If both `zones_fpath` and `zones_dset` are specified, supply curve + aggregation will be applied separately for each zone within each + supply curve site. If specified without `zones_fpath`, a warning + will be logged. Examples -------- @@ -708,6 +724,8 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, self._friction_dset = friction_dset self._data_layers = data_layers self._recalc_lcoe = recalc_lcoe + self._zones_fpath = zones_fpath + self._zones_dset = zones_dset logger.debug("Resource class bins: {}".format(self._res_class_bins)) @@ -721,6 +739,22 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, logger.warning(msg) warn(msg, InputWarning) + if self._zones_fpath is not None and self._zones_dset is None: + msg = ( + "zones_fpath specified without zones_dset. Supply curve " + "aggregation will be performed without zones." + ) + logger.warning(msg) + warn(msg, InputWarning) + + if self._zones_dset is not None and self._zones_fpath is None: + msg = ( + "_zones_dset specified without _zones_fpath. Supply curve " + "aggregation will be performed without zones." + ) + logger.warning(msg) + warn(msg, InputWarning) + self._check_data_layers() def _check_data_layers( @@ -950,6 +984,8 @@ def run_serial( excl_area=None, cap_cost_scale=None, recalc_lcoe=True, + zones_fpath=None, + zones_dset=None, ): """Standalone method to create agg summary - can be parallelized. @@ -1043,6 +1079,20 @@ def run_serial( datasets to be aggregated in the h5_dsets input: system_capacity, fixed_charge_rate, capital_cost, fixed_operating_cost, and variable_operating_cost. + zones_fpath : str | None, optional + Filepath to HDF5 file containing `zones_dset`. If both + `zones_fpath` and `zones_dset` are specified, supply curve + aggregation will be applied separately for each zone within each + supply curve site. This file should match the format of a typical + exclusions HDF5 file. + zones_dset: str | None, optional + Dataset name in the `zones_fpath` file containing the zones to be + applied. This data layer should consist of unique integer values + for each zone, and should be consistent in shape with datasets in + `excl_fpath`. Values of zero will be treated as no data /ignored. + Ifboth `zones_fpath` and `zones_dset` are specified, supply curve + aggregation will be applied separately for each zone within each + supply curve site. Returns ------- @@ -1093,45 +1143,55 @@ def run_serial( inclusion_mask, gid, slice_lookup, resolution=resolution ) - for ri, res_bin in enumerate(res_class_bins): - try: - pointsum = GenerationSupplyCurvePoint.summarize( - gid, - fh.exclusions, - fh.gen, - tm_dset, - gen_index, - res_class_dset=res_data, - res_class_bin=res_bin, - cf_dset=cf_dset, - lcoe_dset=lcoe_data, - h5_dsets=h5_dsets_data, - data_layers=fh.data_layers, - resolution=resolution, - exclusion_shape=exclusion_shape, - power_density=fh.power_density, - args=args, - excl_dict=excl_dict, - inclusion_mask=gid_inclusions, - excl_area=excl_area, - close=False, - friction_layer=fh.friction_layer, - cap_cost_scale=cap_cost_scale, - recalc_lcoe=recalc_lcoe, - ) + zones = cls._get_gid_zones( + zones_fpath, zones_dset, gid, slice_lookup + ) + zone_ids = np.unique(zones).tolist() - except EmptySupplyCurvePointError: - logger.debug("SC point {} is empty".format(gid)) - else: - pointsum['res_class'] = ri + for ri, res_bin in enumerate(res_class_bins): + for zone_id in zone_ids: + zone_mask = zones == zone_id + try: + pointsum = GenerationSupplyCurvePoint.summarize( + gid, + fh.exclusions, + fh.gen, + tm_dset, + gen_index, + res_class_dset=res_data, + res_class_bin=res_bin, + cf_dset=cf_dset, + lcoe_dset=lcoe_data, + h5_dsets=h5_dsets_data, + data_layers=fh.data_layers, + resolution=resolution, + exclusion_shape=exclusion_shape, + power_density=fh.power_density, + args=args, + excl_dict=excl_dict, + inclusion_mask=gid_inclusions, + excl_area=excl_area, + close=False, + friction_layer=fh.friction_layer, + cap_cost_scale=cap_cost_scale, + recalc_lcoe=recalc_lcoe, + zone_mask=zone_mask, + ) - summary.append(pointsum) - logger.debug( - "Serial aggregation completed gid {}: " - "{} out of {} points complete".format( - gid, n_finished, len(gids) + except EmptySupplyCurvePointError: + logger.debug("SC point {} is empty".format(gid)) + else: + pointsum['res_class'] = ri + + summary.append(pointsum) + logger.debug( + "Serial aggregation completed gid {}, " + "resource class {}, zone ID {}: " + "{} out of {} points complete".format( + gid, ri, zone_id, n_finished, + len(gids) + ) ) - ) n_finished += 1 @@ -1227,6 +1287,8 @@ def run_parallel( excl_area=self._excl_area, cap_cost_scale=self._cap_cost_scale, recalc_lcoe=self._recalc_lcoe, + zones_fpath=self._zones_fpath, + zones_dset=self._zones_dset, ) ) @@ -1365,6 +1427,8 @@ def summarize( excl_area=self._excl_area, cap_cost_scale=self._cap_cost_scale, recalc_lcoe=self._recalc_lcoe, + zones_fpath=self._zones_fpath, + zones_dset=self._zones_dset, ) else: summary = self.run_parallel( From e36980c40b81f616f84b4f20f0ed18e1ad671001 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Thu, 16 Jan 2025 17:14:25 -0700 Subject: [PATCH 08/23] adding zone_id to output fields for supply curve aggregation --- reV/supply_curve/sc_aggregation.py | 1 + reV/utilities/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index b71ec98f9..657afa5d4 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -1182,6 +1182,7 @@ def run_serial( logger.debug("SC point {} is empty".format(gid)) else: pointsum['res_class'] = ri + pointsum['zone_id'] = zone_id summary.append(pointsum) logger.debug( diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 793895590..147b313aa 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -135,6 +135,7 @@ class SupplyCurveField(FieldEnum): GEN_GIDS = "gen_gids" GID_COUNTS = "gid_counts" N_GIDS = "n_gids" + ZONE_ID = "zone_id" MEAN_RES = "resource" MEAN_CF_AC = "capacity_factor_ac" MEAN_CF_DC = "capacity_factor_dc" From 5de7124b29f68786a99ffabefac3c96c435b4ba8 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Thu, 16 Jan 2025 17:16:51 -0700 Subject: [PATCH 09/23] adding basic test for aggregation with zones --- tests/test_supply_curve_sc_aggregation.py | 86 ++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index 1beaf8009..b42c157f3 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -29,6 +29,7 @@ from reV.supply_curve.cli_sc_aggregation import _format_res_fpath from reV.handlers.exclusions import LATITUDE from reV.utilities import ModuleName, SupplyCurveField +from reV.supply_curve.extent import SupplyCurveExtent EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') @@ -711,6 +712,89 @@ def test_format_res_fpath_with_year_pattern(): assert _format_res_fpath(config) == {"res_fpath": tf.format(2010)} +@pytest.mark.parametrize("zone_config", ["one_full"]) +def test_agg_zones(zone_config): + """Test sc aggregation with zones within each sc site.""" + # TODO: other test permutations: + # multiple zone configurations: + # single zone (full), single zone (one_partial) 2 zones (two), 3 zones + # (four)? + # run parallel, run serial, run with inclusion mask + # separate test for running via cli + # test with only 1 or the other of the 2 inputs (should produce warnings) + + resolution = 64 + gids = [1, 2, 3] + + with tempfile.TemporaryDirectory() as td: + excl_temp = os.path.join(td, "excl.h5") + shutil.copy(EXCL, excl_temp) + with SupplyCurveExtent(excl_temp, resolution=resolution) as sc: + slice_lookup = sc.get_slice_lookup(gids) + + with h5py.File(excl_temp, 'a') as f: + shape = f[LATITUDE].shape + attrs = dict(f['ri_smod'].attrs) + profile = json.loads(attrs["profile"]) + profile["dtype"] = "uint32" + profile["nodata"] = 0 + attrs["profile"] = json.dumps(profile) + data = np.zeros(shape, dtype=np.uint32) + if zone_config == "one_full": + # each supply curve cell is a single zone, where the zone ID is + # 10 + the gid + for gid, gid_slice in slice_lookup.items(): + data[gid_slice] = gid + 10 + # use the standard test dataset + baseline = AGG_BASELINE + else: + raise NotImplementedError( + "Test for zone_config {zone_config} not yet implemented" + ) + test_dset = "parcels" + f.create_dataset(test_dset, shape, data=data) + for k, v in attrs.items(): + f[test_dset].attrs[k] = v + + sca = SupplyCurveAggregation( + excl_temp, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + zones_fpath=excl_temp, + zones_dset=test_dset, + resolution=resolution, + gids=gids, + ) + summary = sca.summarize(GEN) + + s_baseline = pd.read_csv(baseline) + s_baseline = s_baseline.rename(columns=LEGACY_SC_COL_MAP) + s_baseline = s_baseline.set_index(s_baseline.columns[0]) + s_baseline_subset = s_baseline[ + s_baseline["sc_point_gid"].isin(gids) + ].copy() + list_cols = ["res_gids", "gen_gids", "gid_counts"] + # convert columns containing lists of integers as strings to lists + # of integers + for list_col in list_cols: + s_baseline_subset[list_col] = s_baseline_subset[list_col].apply( + json.loads + ) + + summary = summary.fillna("None") + s_baseline_subset = s_baseline_subset.fillna("None") + + compare_cols = list( + set(s_baseline_subset.columns).intersection(summary.columns) + ) + assert_frame_equal( + summary[compare_cols], + s_baseline_subset[compare_cols], check_dtype=False, rtol=0.0001 + ) + + def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. @@ -723,7 +807,7 @@ def execute_pytest(capture="all", flags="-rapP"): Which tests to show logs and results for. """ - fname = os.path.basename(__file__) + fname = __file__ pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) From ae8a911746e0c46c92fda6a09a289c336ef0eabf Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Mon, 20 Jan 2025 09:41:32 -0700 Subject: [PATCH 10/23] removing separate zones_fpath input, other minor cleanup --- reV/supply_curve/aggregation.py | 10 ++-- reV/supply_curve/points.py | 8 +-- reV/supply_curve/sc_aggregation.py | 65 +++++------------------ tests/test_supply_curve_sc_aggregation.py | 1 - 4 files changed, 22 insertions(+), 62 deletions(-) diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index 1879dc6e4..35c6c9957 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -439,13 +439,13 @@ def _get_gid_inclusion_mask( return gid_inclusions @staticmethod - def _get_gid_zones(zones_fpath, zones_dset, gid, slice_lookup): + def _get_gid_zones(excl_fpath, zones_dset, gid, slice_lookup): """ Get zones 2D array for desired gid. Parameters ---------- - zones_fpath : str | None, optional + excl_fpath : str | None, optional Filepath to HDF5 file containing `zones_dset`. If not specified, output of function will be an array containing all values equal to 1. @@ -466,12 +466,12 @@ def _get_gid_zones(zones_fpath, zones_dset, gid, slice_lookup): """ row_slice, col_slice = slice_lookup[gid] - if zones_fpath is not None and zones_dset is not None: - with ExclusionLayers(zones_fpath) as fh: + if excl_fpath is not None and zones_dset is not None: + with ExclusionLayers(excl_fpath) as fh: if zones_dset not in fh: msg = ( f"Could not find zones_dset {zones_dset} in " - f"zones_fpath {zones_fpath}." + f"zones_fpath {excl_fpath}." ) logger.error(msg) raise FileInputError(msg) diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 975e37cbc..6e99f5aa7 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -474,11 +474,11 @@ def zone_mask(self): if self._zone_mask is None: return None - else: - out_of_extent = self._gids.reshape(self._zone_mask.shape) == -1 - self._zone_mask[out_of_extent] = 0.0 - return self._zone_mask + out_of_extent = self._gids.reshape(self._zone_mask.shape) == -1 + self._zone_mask[out_of_extent] = 0.0 + + return self._zone_mask @property def include_mask(self): diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index 657afa5d4..ade1136c2 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -252,8 +252,7 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, res_class_bins=None, cf_dset='cf_mean-means', lcoe_dset='lcoe_fcr-means', h5_dsets=None, data_layers=None, power_density=None, friction_fpath=None, friction_dset=None, - cap_cost_scale=None, recalc_lcoe=True, zones_fpath=None, - zones_dset=None): + cap_cost_scale=None, recalc_lcoe=True, zones_dset=None): r"""ReV supply curve points aggregation framework. ``reV`` supply curve aggregation combines a high-resolution @@ -544,22 +543,12 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, generation HDF5 output, or if `recalc_lcoe` is set to ``False``, the mean LCOE will be computed from the data stored under the `lcoe_dset` instead. By default, ``True``. - zones_fpath : str, optional - Filepath to HDF5 file containing `zones_dset`. If both - `zones_fpath` and `zones_dset` are specified, supply curve - aggregation will be applied separately for each zone within each - supply curve site. This file should match the format of a typical - exclusions HDF5 file. If specified without zones_dset, a warning - will be logged. zones_dset: str, optoinal - Dataset name in the `zones_fpath` file containing the zones to be - applied. This data layer should consist of unique integer values - for each zone, and should be consistent in shape with datasets in - `excl_fpath`. Values of zero will be treated as no data /ignored. - If both `zones_fpath` and `zones_dset` are specified, supply curve - aggregation will be applied separately for each zone within each - supply curve site. If specified without `zones_fpath`, a warning - will be logged. + Dataset name in `excl_fpath` containing the zones to be applied. + This data layer should consist of unique integer values for each + zone. Values of zero will be treated as no data /ignored. If + `zones_dset` is provided, supply curve aggregation will be applied + separately for each zone within each supply curve site. Examples -------- @@ -724,7 +713,6 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, self._friction_dset = friction_dset self._data_layers = data_layers self._recalc_lcoe = recalc_lcoe - self._zones_fpath = zones_fpath self._zones_dset = zones_dset logger.debug("Resource class bins: {}".format(self._res_class_bins)) @@ -739,22 +727,6 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, logger.warning(msg) warn(msg, InputWarning) - if self._zones_fpath is not None and self._zones_dset is None: - msg = ( - "zones_fpath specified without zones_dset. Supply curve " - "aggregation will be performed without zones." - ) - logger.warning(msg) - warn(msg, InputWarning) - - if self._zones_dset is not None and self._zones_fpath is None: - msg = ( - "_zones_dset specified without _zones_fpath. Supply curve " - "aggregation will be performed without zones." - ) - logger.warning(msg) - warn(msg, InputWarning) - self._check_data_layers() def _check_data_layers( @@ -984,7 +956,6 @@ def run_serial( excl_area=None, cap_cost_scale=None, recalc_lcoe=True, - zones_fpath=None, zones_dset=None, ): """Standalone method to create agg summary - can be parallelized. @@ -1079,20 +1050,12 @@ def run_serial( datasets to be aggregated in the h5_dsets input: system_capacity, fixed_charge_rate, capital_cost, fixed_operating_cost, and variable_operating_cost. - zones_fpath : str | None, optional - Filepath to HDF5 file containing `zones_dset`. If both - `zones_fpath` and `zones_dset` are specified, supply curve - aggregation will be applied separately for each zone within each - supply curve site. This file should match the format of a typical - exclusions HDF5 file. - zones_dset: str | None, optional - Dataset name in the `zones_fpath` file containing the zones to be - applied. This data layer should consist of unique integer values - for each zone, and should be consistent in shape with datasets in - `excl_fpath`. Values of zero will be treated as no data /ignored. - Ifboth `zones_fpath` and `zones_dset` are specified, supply curve - aggregation will be applied separately for each zone within each - supply curve site. + zones_dset: str, optoinal + Dataset name in `excl_fpath` containing the zones to be applied. + This data layer should consist of unique integer values for each + zone. Values of zero will be treated as no data /ignored. If + `zones_dset` is provided, supply curve aggregation will be applied + separately for each zone within each supply curve site. Returns ------- @@ -1144,7 +1107,7 @@ def run_serial( ) zones = cls._get_gid_zones( - zones_fpath, zones_dset, gid, slice_lookup + excl_fpath, zones_dset, gid, slice_lookup ) zone_ids = np.unique(zones).tolist() @@ -1288,7 +1251,6 @@ def run_parallel( excl_area=self._excl_area, cap_cost_scale=self._cap_cost_scale, recalc_lcoe=self._recalc_lcoe, - zones_fpath=self._zones_fpath, zones_dset=self._zones_dset, ) ) @@ -1428,7 +1390,6 @@ def summarize( excl_area=self._excl_area, cap_cost_scale=self._cap_cost_scale, recalc_lcoe=self._recalc_lcoe, - zones_fpath=self._zones_fpath, zones_dset=self._zones_dset, ) else: diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index b42c157f3..f64e21eab 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -762,7 +762,6 @@ def test_agg_zones(zone_config): excl_dict=EXCL_DICT, res_class_dset=RES_CLASS_DSET, res_class_bins=RES_CLASS_BINS, - zones_fpath=excl_temp, zones_dset=test_dset, resolution=resolution, gids=gids, From 369a4e42bb259101fe0c251727e1edfad30df2f3 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Mon, 20 Jan 2025 09:58:21 -0700 Subject: [PATCH 11/23] trailing change from prior commit switching zones_fpath to excl_fpath --- reV/supply_curve/aggregation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index 35c6c9957..ca30d0b4c 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -450,7 +450,7 @@ def _get_gid_zones(excl_fpath, zones_dset, gid, slice_lookup): output of function will be an array containing all values equal to 1. zones_dset : str | None, optional - Dataset name in the `zones_fpath` file containing the zones to be + Dataset name in the `excl_fpath` file containing the zones to be loaded. If not specified, output of function will be an array containing all values equal to 1. gid : int @@ -471,7 +471,7 @@ def _get_gid_zones(excl_fpath, zones_dset, gid, slice_lookup): if zones_dset not in fh: msg = ( f"Could not find zones_dset {zones_dset} in " - f"zones_fpath {excl_fpath}." + f"excl_fpath {excl_fpath}." ) logger.error(msg) raise FileInputError(msg) From ef0559930186abd526a7199253304be17f414d67 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Mon, 20 Jan 2025 09:58:52 -0700 Subject: [PATCH 12/23] removing check of zones_mask for values of 0 and 1 since input is typically/likely boolean --- reV/supply_curve/points.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 6e99f5aa7..ebf94da29 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -257,8 +257,8 @@ def __init__( Flag to close object file handlers on exit. zone_mask : np.ndarray | None, optional 2D array defining zone within the supply curve to be evaluated, - where 1 is included and 0 is excluded. The shape of this will be - checked against the input resolution. + where values of 0 or False will be excluded. The shape of this + array will be checked against the input resolution. """ self._excl_dict = excl_dict @@ -595,14 +595,6 @@ def _check_zone_mask(self): assert self._zone_mask.shape[1] <= self._resolution, msg assert self._zone_mask.size == len(self._gids), msg - if not np.isin(self._zone_mask, [0, 1]).all(): - msg = ( - "zone_mask includes unexpected values. All values must be " - "in the domain: [0, 1]" - ) - logger.error(msg) - raise ValueError(msg) - def exclusion_weighted_mean(self, arr, drop_nan=True): """ Calc the exclusions-weighted mean value of an array of resource data. From 9d36e5f3ac01825fb83ab885401a1a35ab8c538a Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Mon, 20 Jan 2025 15:01:27 -0700 Subject: [PATCH 13/23] adding new test case for test_agg_zones, related fixes --- reV/supply_curve/points.py | 2 +- reV/supply_curve/sc_aggregation.py | 2 +- ...baseline_agg_summary_zones_one_partial.csv | 4 ++ tests/test_supply_curve_sc_aggregation.py | 43 +++++++++++++++---- 4 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 tests/data/sc_out/baseline_agg_summary_zones_one_partial.csv diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index ebf94da29..52472a03d 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -287,7 +287,7 @@ def __init__( assert inclusion_mask.size == len(self._gids), msg self._incl_mask = inclusion_mask.copy() - self._zone_mask = zone_mask + self._zone_mask = zone_mask.copy() self._check_zone_mask() self._centroid = None diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index ade1136c2..c60bdb661 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -1109,7 +1109,7 @@ def run_serial( zones = cls._get_gid_zones( excl_fpath, zones_dset, gid, slice_lookup ) - zone_ids = np.unique(zones).tolist() + zone_ids = np.unique(zones[zones != 0]).tolist() for ri, res_bin in enumerate(res_class_bins): for zone_id in zone_ids: diff --git a/tests/data/sc_out/baseline_agg_summary_zones_one_partial.csv b/tests/data/sc_out/baseline_agg_summary_zones_one_partial.csv new file mode 100644 index 000000000..e1a4a55d4 --- /dev/null +++ b/tests/data/sc_out/baseline_agg_summary_zones_one_partial.csv @@ -0,0 +1,4 @@ +sc_gid,latitude,longitude,country,state,county,elevation_m,timezone,sc_point_gid,sc_row_ind,sc_col_ind,res_gids,gen_gids,gid_counts,n_gids,offshore,capacity_factor_ac,capacity_factor_dc,lcoe_site_usd_per_mwh,resource,area_developable_sq_km,capacity_ac_mw,capacity_dc_mw,multiplier_cc_eos,multiplier_cc_regional,annual_energy_site_mwh,cost_site_occ_usd_per_ac_mw,cost_base_occ_usd_per_ac_mw,cost_site_foc_usd_per_ac_mw,cost_base_foc_usd_per_ac_mw,cost_site_voc_usd_per_ac_mw,cost_base_voc_usd_per_ac_mw,fixed_charge_rate,res_class,zone_id +0,41.99292076811808,-71.80788736558492,United States,Rhode Island,Providence,194.64,-5,1,0,1,[1316631],[7],[12.0],12,,0.12399998,,124.00283,3.9910002,0.0972,3.4991999999999996,,1,1,3800.970489114761,,,,,,,0,0,11 +1,41.993086159675855,-71.73837273978268,United States,Rhode Island,Providence,135.3,-5,2,0,2,[1318507],[40],[14.0],14,,0.123999976,,123.88893,4.001,0.1134,4.0824,,1,1,4434.465304187536,,,,,,,0,0,12 +2,41.993209457643076,-71.66885760530704,United States,Rhode Island,Providence,116.0,-5,3,0,3,[1321872],[76],[6.0],6,,0.124000005,,123.658875,4.0159993,0.0486,1.7495999999999998,,1,1,1900.4855871312616,,,,,,,0,0,13 diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index f64e21eab..635b08561 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -712,12 +712,15 @@ def test_format_res_fpath_with_year_pattern(): assert _format_res_fpath(config) == {"res_fpath": tf.format(2010)} -@pytest.mark.parametrize("zone_config", ["one_full"]) +@pytest.mark.parametrize("zone_config", [ + "one_full", + "one_partial", +]) def test_agg_zones(zone_config): """Test sc aggregation with zones within each sc site.""" # TODO: other test permutations: # multiple zone configurations: - # single zone (full), single zone (one_partial) 2 zones (two), 3 zones + # single zone (one_partial) 2 zones (two), 3 zones # (four)? # run parallel, run serial, run with inclusion mask # separate test for running via cli @@ -741,16 +744,36 @@ def test_agg_zones(zone_config): attrs["profile"] = json.dumps(profile) data = np.zeros(shape, dtype=np.uint32) if zone_config == "one_full": - # each supply curve cell is a single zone, where the zone ID is - # 10 + the gid for gid, gid_slice in slice_lookup.items(): data[gid_slice] = gid + 10 # use the standard test dataset baseline = AGG_BASELINE + excl_dict = EXCL_DICT.copy() + res_class_bins = RES_CLASS_BINS + apply_legacy_remap = True else: - raise NotImplementedError( - "Test for zone_config {zone_config} not yet implemented" + excl_dict = { + k: v for k, v in EXCL_DICT.items() if k == "ri_srtm_slope" + } + res_class_bins = None + baseline = os.path.join( + TESTDATADIR, + f"sc_out/baseline_agg_summary_zones_{zone_config}.csv" ) + apply_legacy_remap = False + if zone_config == "one_partial": + for gid, gid_slice in slice_lookup.items(): + gid_rows, gid_cols = gid_slice + zone_rows = slice(gid_rows.stop - 4, gid_rows.stop) + zone_cols = slice(gid_cols.stop - 4, gid_cols.stop) + data[(zone_rows, zone_cols)] = gid + 10 + elif zone_config == "two": + for gid, gid_slice in slice_lookup.items(): + gid_rows, gid_cols = gid_slice + zone_rows = slice(gid_rows.stop - 4, gid_rows.stop) + zone_cols = slice(gid_cols.stop - 4, gid_cols.stop) + data[(zone_rows, zone_cols)] = gid + 10 + test_dset = "parcels" f.create_dataset(test_dset, shape, data=data) for k, v in attrs.items(): @@ -759,17 +782,19 @@ def test_agg_zones(zone_config): sca = SupplyCurveAggregation( excl_temp, TM_DSET, - excl_dict=EXCL_DICT, + excl_dict=excl_dict, res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, + res_class_bins=res_class_bins, zones_dset=test_dset, resolution=resolution, + power_density=36.0, gids=gids, ) summary = sca.summarize(GEN) s_baseline = pd.read_csv(baseline) - s_baseline = s_baseline.rename(columns=LEGACY_SC_COL_MAP) + if apply_legacy_remap: + s_baseline = s_baseline.rename(columns=LEGACY_SC_COL_MAP) s_baseline = s_baseline.set_index(s_baseline.columns[0]) s_baseline_subset = s_baseline[ s_baseline["sc_point_gid"].isin(gids) From 7fe92366b5563e30b3e8fc1c9c474fbd7077f8e8 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Mon, 20 Jan 2025 15:28:04 -0700 Subject: [PATCH 14/23] adding more test cases to test_agg_zones --- ...l.csv => baseline_agg_summary_zones_1.csv} | 0 .../sc_out/baseline_agg_summary_zones_2.csv | 7 ++ .../sc_out/baseline_agg_summary_zones_3.csv | 10 +++ tests/test_supply_curve_sc_aggregation.py | 75 ++++++++----------- 4 files changed, 50 insertions(+), 42 deletions(-) rename tests/data/sc_out/{baseline_agg_summary_zones_one_partial.csv => baseline_agg_summary_zones_1.csv} (100%) create mode 100644 tests/data/sc_out/baseline_agg_summary_zones_2.csv create mode 100644 tests/data/sc_out/baseline_agg_summary_zones_3.csv diff --git a/tests/data/sc_out/baseline_agg_summary_zones_one_partial.csv b/tests/data/sc_out/baseline_agg_summary_zones_1.csv similarity index 100% rename from tests/data/sc_out/baseline_agg_summary_zones_one_partial.csv rename to tests/data/sc_out/baseline_agg_summary_zones_1.csv diff --git a/tests/data/sc_out/baseline_agg_summary_zones_2.csv b/tests/data/sc_out/baseline_agg_summary_zones_2.csv new file mode 100644 index 000000000..0e58f79de --- /dev/null +++ b/tests/data/sc_out/baseline_agg_summary_zones_2.csv @@ -0,0 +1,7 @@ +sc_gid,latitude,longitude,country,state,county,elevation_m,timezone,sc_point_gid,sc_row_ind,sc_col_ind,res_gids,gen_gids,gid_counts,n_gids,offshore,capacity_factor_ac,capacity_factor_dc,lcoe_site_usd_per_mwh,resource,area_developable_sq_km,capacity_ac_mw,capacity_dc_mw,multiplier_cc_eos,multiplier_cc_regional,annual_energy_site_mwh,cost_site_occ_usd_per_ac_mw,cost_base_occ_usd_per_ac_mw,cost_site_foc_usd_per_ac_mw,cost_base_foc_usd_per_ac_mw,cost_site_voc_usd_per_ac_mw,cost_base_voc_usd_per_ac_mw,fixed_charge_rate,res_class,zone_id +0,41.99292076811808,-71.80788736558492,United States,Rhode Island,Providence,194.64,-5,1,0,1,[1316631],[7],[12.0],12,,0.12399998,,124.00283,3.9910002,0.0972,3.4991999999999996,,1,1,3800.970489114761,,,,,,,0,0,11 +1,41.99292076811808,-71.80788736558492,United States,Rhode Island,Providence,194.64,-5,1,0,1,[1316631],[7],[13.0],13,,0.12399998,,124.00282,3.9910002,0.10529999999999999,3.7907999999999995,,1,1,4117.718029874324,,,,,,,0,0,21 +2,41.993086159675855,-71.73837273978268,United States,Rhode Island,Providence,135.3,-5,2,0,2,[1318507],[40],[14.0],14,,0.123999976,,123.88893,4.001,0.1134,4.0824,,1,1,4434.465304187536,,,,,,,0,0,12 +3,41.993086159675855,-71.73837273978268,United States,Rhode Island,Providence,135.3,-5,2,0,2,[1318507],[40],[14.0],14,,0.123999976,,123.88893,4.001,0.1134,4.0824,,1,1,4434.465304187536,,,,,,,0,0,22 +4,41.993209457643076,-71.66885760530704,United States,Rhode Island,Providence,116.0,-5,3,0,3,[1321872],[76],[6.0],6,,0.124000005,,123.658875,4.0159993,0.0486,1.7495999999999998,,1,1,1900.4855871312616,,,,,,,0,0,13 +5,41.993209457643076,-71.66885760530704,United States,Rhode Island,Providence,116.0,-5,3,0,3,[1321872],[76],[1.0],1,,0.124,,123.658875,4.016,0.0081,0.29159999999999997,,1,1,316.747578823328,,,,,,,0,0,23 diff --git a/tests/data/sc_out/baseline_agg_summary_zones_3.csv b/tests/data/sc_out/baseline_agg_summary_zones_3.csv new file mode 100644 index 000000000..2bea5721a --- /dev/null +++ b/tests/data/sc_out/baseline_agg_summary_zones_3.csv @@ -0,0 +1,10 @@ +sc_gid,latitude,longitude,country,state,county,elevation_m,timezone,sc_point_gid,sc_row_ind,sc_col_ind,res_gids,gen_gids,gid_counts,n_gids,offshore,capacity_factor_ac,capacity_factor_dc,lcoe_site_usd_per_mwh,resource,area_developable_sq_km,capacity_ac_mw,capacity_dc_mw,multiplier_cc_eos,multiplier_cc_regional,annual_energy_site_mwh,cost_site_occ_usd_per_ac_mw,cost_base_occ_usd_per_ac_mw,cost_site_foc_usd_per_ac_mw,cost_base_foc_usd_per_ac_mw,cost_site_voc_usd_per_ac_mw,cost_base_voc_usd_per_ac_mw,fixed_charge_rate,res_class,zone_id +0,41.99292076811808,-71.80788736558492,United States,Rhode Island,Providence,194.64,-5,1,0,1,[1316631],[7],[12.0],12,,0.12399998,,124.00283,3.9910002,0.0972,3.4991999999999996,,1,1,3800.970489114761,,,,,,,0,0,11 +1,41.99292076811808,-71.80788736558492,United States,Rhode Island,Providence,194.64,-5,1,0,1,[1316631],[7],[13.0],13,,0.12399998,,124.00282,3.9910002,0.10529999999999999,3.7907999999999995,,1,1,4117.718029874324,,,,,,,0,0,21 +2,41.99292076811808,-71.80788736558492,United States,Rhode Island,Providence,194.64,-5,1,0,1,[1316631],[7],[13.0],13,,0.12399998,,124.00282,3.9910002,0.10529999999999999,3.7907999999999995,,1,1,4117.718029874324,,,,,,,0,0,31 +3,41.993086159675855,-71.73837273978268,United States,Rhode Island,Providence,135.3,-5,2,0,2,[1318507],[40],[14.0],14,,0.123999976,,123.88893,4.001,0.1134,4.0824,,1,1,4434.465304187536,,,,,,,0,0,12 +4,41.993086159675855,-71.73837273978268,United States,Rhode Island,Providence,135.3,-5,2,0,2,[1318507],[40],[14.0],14,,0.123999976,,123.88893,4.001,0.1134,4.0824,,1,1,4434.465304187536,,,,,,,0,0,22 +5,41.993086159675855,-71.73837273978268,United States,Rhode Island,Providence,135.3,-5,2,0,2,[1318507],[40],[11.0],11,,0.12399998,,123.88894,4.001,0.0891,3.2076,,1,1,3484.222948355198,,,,,,,0,0,32 +6,41.993209457643076,-71.66885760530704,United States,Rhode Island,Providence,116.0,-5,3,0,3,[1321872],[76],[6.0],6,,0.124000005,,123.658875,4.0159993,0.0486,1.7495999999999998,,1,1,1900.4855871312616,,,,,,,0,0,13 +7,41.993209457643076,-71.66885760530704,United States,Rhode Island,Providence,116.0,-5,3,0,3,[1321872],[76],[1.0],1,,0.124,,123.658875,4.016,0.0081,0.29159999999999997,,1,1,316.747578823328,,,,,,,0,0,23 +8,41.993209457643076,-71.66885760530704,United States,Rhode Island,Providence,116.0,-5,3,0,3,[1321872],[76],[3.0],3,,0.12399999,,123.658875,4.016,0.0243,0.8747999999999999,,1,1,950.2426793743372,,,,,,,0,0,33 diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index 635b08561..4ee869dd7 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -712,19 +712,12 @@ def test_format_res_fpath_with_year_pattern(): assert _format_res_fpath(config) == {"res_fpath": tf.format(2010)} -@pytest.mark.parametrize("zone_config", [ - "one_full", - "one_partial", -]) +@pytest.mark.parametrize("zone_config", ["one_full", 1, 2, 3]) def test_agg_zones(zone_config): """Test sc aggregation with zones within each sc site.""" # TODO: other test permutations: - # multiple zone configurations: - # single zone (one_partial) 2 zones (two), 3 zones - # (four)? # run parallel, run serial, run with inclusion mask # separate test for running via cli - # test with only 1 or the other of the 2 inputs (should produce warnings) resolution = 64 gids = [1, 2, 3] @@ -744,6 +737,7 @@ def test_agg_zones(zone_config): attrs["profile"] = json.dumps(profile) data = np.zeros(shape, dtype=np.uint32) if zone_config == "one_full": + # each entire cell is one zone for gid, gid_slice in slice_lookup.items(): data[gid_slice] = gid + 10 # use the standard test dataset @@ -761,18 +755,15 @@ def test_agg_zones(zone_config): f"sc_out/baseline_agg_summary_zones_{zone_config}.csv" ) apply_legacy_remap = False - if zone_config == "one_partial": - for gid, gid_slice in slice_lookup.items(): - gid_rows, gid_cols = gid_slice - zone_rows = slice(gid_rows.stop - 4, gid_rows.stop) - zone_cols = slice(gid_cols.stop - 4, gid_cols.stop) - data[(zone_rows, zone_cols)] = gid + 10 - elif zone_config == "two": - for gid, gid_slice in slice_lookup.items(): - gid_rows, gid_cols = gid_slice - zone_rows = slice(gid_rows.stop - 4, gid_rows.stop) + for gid, gid_slice in slice_lookup.items(): + gid_rows, gid_cols = gid_slice + for z in range(0, zone_config): + zone_rows = slice( + gid_rows.stop - (z + 1) * 4, + gid_rows.stop - z * 4 + ) zone_cols = slice(gid_cols.stop - 4, gid_cols.stop) - data[(zone_rows, zone_cols)] = gid + 10 + data[(zone_rows, zone_cols)] = gid + 10 * (z + 1) test_dset = "parcels" f.create_dataset(test_dset, shape, data=data) @@ -792,31 +783,31 @@ def test_agg_zones(zone_config): ) summary = sca.summarize(GEN) - s_baseline = pd.read_csv(baseline) - if apply_legacy_remap: - s_baseline = s_baseline.rename(columns=LEGACY_SC_COL_MAP) - s_baseline = s_baseline.set_index(s_baseline.columns[0]) - s_baseline_subset = s_baseline[ - s_baseline["sc_point_gid"].isin(gids) - ].copy() - list_cols = ["res_gids", "gen_gids", "gid_counts"] - # convert columns containing lists of integers as strings to lists - # of integers - for list_col in list_cols: - s_baseline_subset[list_col] = s_baseline_subset[list_col].apply( - json.loads - ) + s_baseline = pd.read_csv(baseline) + if apply_legacy_remap: + s_baseline = s_baseline.rename(columns=LEGACY_SC_COL_MAP) + s_baseline = s_baseline.set_index(s_baseline.columns[0]) + s_baseline_subset = s_baseline[ + s_baseline["sc_point_gid"].isin(gids) + ].copy() + list_cols = ["res_gids", "gen_gids", "gid_counts"] + # convert columns containing lists of integers as strings to lists + # of integers + for list_col in list_cols: + s_baseline_subset[list_col] = s_baseline_subset[list_col].apply( + json.loads + ) - summary = summary.fillna("None") - s_baseline_subset = s_baseline_subset.fillna("None") + summary = summary.fillna("None") + s_baseline_subset = s_baseline_subset.fillna("None") - compare_cols = list( - set(s_baseline_subset.columns).intersection(summary.columns) - ) - assert_frame_equal( - summary[compare_cols], - s_baseline_subset[compare_cols], check_dtype=False, rtol=0.0001 - ) + compare_cols = list( + set(s_baseline_subset.columns).intersection(summary.columns) + ) + assert_frame_equal( + summary[compare_cols], + s_baseline_subset[compare_cols], check_dtype=False, rtol=0.0001 + ) def execute_pytest(capture="all", flags="-rapP"): From 8de4a902d1ed731e91bf7e8cfb1ac6da454c36a3 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Mon, 20 Jan 2025 15:48:36 -0700 Subject: [PATCH 15/23] adding more test cases for test_agg_zones --- tests/test_supply_curve_sc_aggregation.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index 4ee869dd7..1f0ccb224 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -712,12 +712,16 @@ def test_format_res_fpath_with_year_pattern(): assert _format_res_fpath(config) == {"res_fpath": tf.format(2010)} -@pytest.mark.parametrize("zone_config", ["one_full", 1, 2, 3]) -def test_agg_zones(zone_config): +@pytest.mark.parametrize("zone_config,max_workers,pre_extract_inclusions", [ + ("one_full", None, False), + (1, None, False), + (2, None, False), + (3, None, False), + (1, 1, False), # test with run_serial + (1, None, True), # test with pre_extract_exclusions +]) +def test_agg_zones(zone_config, max_workers, pre_extract_inclusions): """Test sc aggregation with zones within each sc site.""" - # TODO: other test permutations: - # run parallel, run serial, run with inclusion mask - # separate test for running via cli resolution = 64 gids = [1, 2, 3] @@ -780,8 +784,9 @@ def test_agg_zones(zone_config): resolution=resolution, power_density=36.0, gids=gids, + pre_extract_inclusions=pre_extract_inclusions, ) - summary = sca.summarize(GEN) + summary = sca.summarize(GEN, max_workers=max_workers) s_baseline = pd.read_csv(baseline) if apply_legacy_remap: From ef9db354e52ac2836b86340f020f44c4121f1dce Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Mon, 20 Jan 2025 15:59:32 -0700 Subject: [PATCH 16/23] fixing zone_mask copy issue when zone_mask is None, removing _check_zone_mask() and replacing with logic in __init__ --- reV/supply_curve/points.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 52472a03d..3f83c4d51 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -287,8 +287,17 @@ def __init__( assert inclusion_mask.size == len(self._gids), msg self._incl_mask = inclusion_mask.copy() - self._zone_mask = zone_mask.copy() - self._check_zone_mask() + self._zone_mask = zone_mask + if zone_mask is not None: + msg = ( + "Bad zone mask input shape of {} with stated " + "resolution of {}".format(zone_mask.shape, resolution) + ) + assert len(zone_mask.shape) == 2, msg + assert zone_mask.shape[0] <= resolution, msg + assert zone_mask.shape[1] <= resolution, msg + assert zone_mask.size == len(self._gids), msg + self._zone_mask = zone_mask.copy() self._centroid = None self._excl_area = excl_area @@ -578,23 +587,6 @@ def _check_excl(self): ) raise EmptySupplyCurvePointError(msg) - def _check_zone_mask(self): - """ - Check that the zone mask is the correct size and shape, and that it - contains only values of 0 and 1. - """ - if self._zone_mask is not None: - msg = ( - "Bad zone mask input shape of {} with stated " - "resolution of {}".format( - self._zone_mask.shape, self._resolution - ) - ) - assert len(self._zone_mask.shape) == 2, msg - assert self._zone_mask.shape[0] <= self._resolution, msg - assert self._zone_mask.shape[1] <= self._resolution, msg - assert self._zone_mask.size == len(self._gids), msg - def exclusion_weighted_mean(self, arr, drop_nan=True): """ Calc the exclusions-weighted mean value of an array of resource data. From 0424beb8b5b74c1d7cf5e11cecf06cb689d8d71b Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Mon, 20 Jan 2025 16:22:55 -0700 Subject: [PATCH 17/23] adding test for sc aggregation with zones via the CLI --- tests/test_supply_curve_sc_aggregation.py | 94 +++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index 1f0ccb224..c5ecd9356 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -815,6 +815,100 @@ def test_agg_zones(zone_config, max_workers, pre_extract_inclusions): ) +def test_cli_agg_zones(runner, clear_loggers): + """ + Test SC aggregation with zones within each SC site via the CLI + """ + + resolution = 64 + gids = [1, 2, 3] + + with tempfile.TemporaryDirectory() as td: + excl_temp = os.path.join(td, "excl.h5") + shutil.copy(EXCL, excl_temp) + with SupplyCurveExtent(excl_temp, resolution=resolution) as sc: + slice_lookup = sc.get_slice_lookup(gids) + + with h5py.File(excl_temp, 'a') as f: + shape = f[LATITUDE].shape + attrs = dict(f['ri_smod'].attrs) + profile = json.loads(attrs["profile"]) + profile["dtype"] = "uint32" + profile["nodata"] = 0 + attrs["profile"] = json.dumps(profile) + data = np.zeros(shape, dtype=np.uint32) + # each entire cell is one zone + for gid, gid_slice in slice_lookup.items(): + data[gid_slice] = gid + 10 + test_dset = "parcels" + f.create_dataset(test_dset, shape, data=data) + for k, v in attrs.items(): + f[test_dset].attrs[k] = v + + config = { + "log_directory": td, + "execution_control": { + "option": "local", + "max_workers": 1, + }, + "log_level": "INFO", + "excl_fpath": excl_temp, + "gen_fpath": GEN, + "econ_fpath": None, + "tm_dset": TM_DSET, + "res_fpath": None, + "res_class_dset": RES_CLASS_DSET, + "res_class_bins": RES_CLASS_BINS, + "excl_dict": EXCL_DICT, + "resolution": resolution, + "zones_dset": test_dset, + "pre_extract_inclusions": False, + "gids": gids, + "power_density": 36, + } + + config_path = os.path.join(td, "config.json") + with open(config_path, "w") as f: + json.dump(config, f) + + result = runner.invoke( + main, [ModuleName.SUPPLY_CURVE_AGGREGATION, "-c", config_path] + ) + clear_loggers() + + if result.exit_code != 0: + msg = "Failed with error {}".format( + traceback.print_exception(*result.exc_info) + ) + raise RuntimeError(msg) + + fn_list = os.listdir(td) + dirname = os.path.basename(td) + out_csv_fn = "{}_{}.csv".format( + dirname, ModuleName.SUPPLY_CURVE_AGGREGATION + ) + assert out_csv_fn in fn_list + + summary = pd.read_csv(os.path.join(td, out_csv_fn)) + + s_baseline = pd.read_csv(AGG_BASELINE) + s_baseline = s_baseline.rename(columns=LEGACY_SC_COL_MAP) + s_baseline_subset = s_baseline[ + s_baseline["sc_point_gid"].isin(gids) + ].copy() + + summary = summary.fillna("None") + s_baseline_subset = s_baseline_subset.fillna("None") + + compare_cols = list( + set(s_baseline_subset.columns).intersection(summary.columns) + ) + assert_frame_equal( + summary[compare_cols], + s_baseline_subset[compare_cols], check_dtype=False, rtol=0.0001 + ) + + def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. From 99368b2c303dcc81a833157d458c8d24aa076080 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Tue, 21 Jan 2025 10:59:31 -0700 Subject: [PATCH 18/23] removing out_of_extent logic from zone_mask(); changing dummy zones array to uint8 --- reV/supply_curve/aggregation.py | 2 +- reV/supply_curve/points.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index ca30d0b4c..06d6fdf7d 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -481,7 +481,7 @@ def _get_gid_zones(excl_fpath, zones_dset, gid, slice_lookup): row_slice.stop - row_slice.start, col_slice.stop - col_slice.start ) - zones = np.ones(shape, dtype="uint32") + zones = np.ones(shape, dtype="uint8") return zones diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 3f83c4d51..fa66ded40 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -480,13 +480,6 @@ def zone_mask(self): ------- np.ndarray """ - - if self._zone_mask is None: - return None - - out_of_extent = self._gids.reshape(self._zone_mask.shape) == -1 - self._zone_mask[out_of_extent] = 0.0 - return self._zone_mask @property From b46d724861e4835270e4144b0b4e1638c63a35c5 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Tue, 21 Jan 2025 11:19:24 -0700 Subject: [PATCH 19/23] cleaning up docstrings related to zones_dset and zone_mask --- reV/supply_curve/points.py | 23 +++++++++++++---------- reV/supply_curve/sc_aggregation.py | 26 ++++++++++++++++---------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index fa66ded40..74a91781b 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -258,7 +258,8 @@ def __init__( zone_mask : np.ndarray | None, optional 2D array defining zone within the supply curve to be evaluated, where values of 0 or False will be excluded. The shape of this - array will be checked against the input resolution. + array will be checked against the input resolution. If not + specified, no zone mask will be applied. """ self._excl_dict = excl_dict @@ -1032,8 +1033,9 @@ def __init__( initialization. zone_mask : np.ndarray | None, optional 2D array defining zone within the supply curve to be evaluated, - where 1 is included and 0 is excluded. The shape of this will be - checked against the input resolution. + where values of 0 or False will be excluded. The shape of this + array will be checked against the input resolution. If not + specified, no zone mask will be applied. """ super().__init__( gid, @@ -1550,9 +1552,10 @@ def __init__( Flag to apply exclusions to the resource / generation gid's on initialization. zone_mask : np.ndarray | None, optional - 2D array defining zone within the supply curve to be summarized, - where 1 is included and 0 is excluded. The shape of this will be - checked against the input resolution. + 2D array defining zone within the supply curve to be evaluated, + where values of 0 or False will be excluded. The shape of this + array will be checked against the input resolution. If not + specified, no zone mask will be applied. """ self._res_class_dset = res_class_dset @@ -2550,10 +2553,10 @@ def summarize( fixed_charge_rate, capital_cost, fixed_operating_cost, and variable_operating_cost. zone_mask : np.ndarray | None, optional - 2D array defining zone within the supply curve to be summarized, - where 1 is included and 0 is excluded. The shape of this will be - checked against the input resolution. If not specified, no zone - mask will be applied. + 2D array defining zone within the supply curve to be evaluated, + where values of 0 or False will be excluded. The shape of this + array will be checked against the input resolution. If not + specified, no zone mask will be applied. Returns ------- diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index c60bdb661..78c94c317 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -543,12 +543,15 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, generation HDF5 output, or if `recalc_lcoe` is set to ``False``, the mean LCOE will be computed from the data stored under the `lcoe_dset` instead. By default, ``True``. - zones_dset: str, optoinal + zones_dset: str, optional Dataset name in `excl_fpath` containing the zones to be applied. - This data layer should consist of unique integer values for each - zone. Values of zero will be treated as no data /ignored. If - `zones_dset` is provided, supply curve aggregation will be applied - separately for each zone within each supply curve site. + If specified, supply curve aggregation will be performed separately + for each discrete zone within each supply curve site. This option + can be used for uses cases such as subdividing sites by parcel, + such that each parcel within each site is output to a separate + sc_gid. The input data layer should consist of unique integer + values for each zone. Values of zero will be treated as excluded + areas. Examples -------- @@ -1050,12 +1053,15 @@ def run_serial( datasets to be aggregated in the h5_dsets input: system_capacity, fixed_charge_rate, capital_cost, fixed_operating_cost, and variable_operating_cost. - zones_dset: str, optoinal + zones_dset: str, optional Dataset name in `excl_fpath` containing the zones to be applied. - This data layer should consist of unique integer values for each - zone. Values of zero will be treated as no data /ignored. If - `zones_dset` is provided, supply curve aggregation will be applied - separately for each zone within each supply curve site. + If specified, supply curve aggregation will be performed separately + for each discrete zone within each supply curve site. This option + can be used for uses cases such as subdividing sites by parcel, + such that each parcel within each site is output to a separate + sc_gid. The input data layer should consist of unique integer + values for each zone. Values of zero will be treated as excluded + areas. Returns ------- From 381a7b9d005a5a55d1ce19940370b829281f0384 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Tue, 21 Jan 2025 11:29:54 -0700 Subject: [PATCH 20/23] bumping version --- reV/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reV/version.py b/reV/version.py index e1201d2a0..f907e6187 100644 --- a/reV/version.py +++ b/reV/version.py @@ -2,4 +2,4 @@ reV Version number """ -__version__ = "0.9.7" +__version__ = "0.9.8" From 1ecd5e8e73065db1e2e05024390e42dbf228819e Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Fri, 24 Jan 2025 09:03:38 -0700 Subject: [PATCH 21/23] Add zone ID to logger message --- reV/supply_curve/sc_aggregation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index 78c94c317..684ceba19 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -1148,7 +1148,8 @@ def run_serial( ) except EmptySupplyCurvePointError: - logger.debug("SC point {} is empty".format(gid)) + logger.debug("SC point {}, zone ID {} is empty" + .format(gid, zone_id)) else: pointsum['res_class'] = ri pointsum['zone_id'] = zone_id From ca4e8c4b131bbcc193fb30ebefc29186b8da9980 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Fri, 24 Jan 2025 09:21:28 -0700 Subject: [PATCH 22/23] Logger message now only talks about zones --- reV/supply_curve/sc_aggregation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index 684ceba19..9c410578c 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -1156,11 +1156,10 @@ def run_serial( summary.append(pointsum) logger.debug( - "Serial aggregation completed gid {}, " + "Serial aggregation completed for" "resource class {}, zone ID {}: " - "{} out of {} points complete".format( - gid, ri, zone_id, n_finished, - len(gids) + "{:,d} out of {:,d} zones complete".format( + ri, zone_id, zi, len(zone_ids) ) ) From a68e4f689dacac6974ef233c50a6344f91538d30 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Fri, 24 Jan 2025 09:23:02 -0700 Subject: [PATCH 23/23] Add logger message about points --- reV/supply_curve/sc_aggregation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index 9c410578c..b1919b003 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -1118,7 +1118,7 @@ def run_serial( zone_ids = np.unique(zones[zones != 0]).tolist() for ri, res_bin in enumerate(res_class_bins): - for zone_id in zone_ids: + for zi, zone_id in enumerate(zone_ids, start=1): zone_mask = zones == zone_id try: pointsum = GenerationSupplyCurvePoint.summarize( @@ -1164,6 +1164,12 @@ def run_serial( ) n_finished += 1 + logger.debug( + "Serial aggregation completed for gid {}: " + "{:,d} out of {:,d} points complete".format( + gid, n_finished, len(gids) + ) + ) return summary