Skip to content

Commit

Permalink
September 2020 Data Cube Utilities Release 2.20 RC2 - Removed load_si…
Browse files Browse the repository at this point in the history 
…mple() and load_multiplatform()
  • Loading branch information
@jcrattz
jcrattz committed Sep 30, 2020
1 parent a91ffa3 commit 9fa8851
Showing 1 changed file with 0 additions and 219 deletions.
219 changes: 0 additions & 219 deletions data_cube_utilities/dc_load.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,225 +259,6 @@ def xr_set_same_coords(datasets):

## End Combining Data ##

## Load ##

def load_simple(dc, platform, product, frac_res=None, abs_res=None,
load_params={}, masking_params={}, indiv_masks=None):
"""
**This function is DEPRECATED.**
Simplifies loading from the Data Cube by retrieving a dataset along
with its mask. Currently only tested on Landsat data.
Parameters
----------
dc: datacube.api.core.Datacube
The Datacube instance to load data with.
platform, product: str
Strings denoting the platform and product to retrieve data for.
frac_res: float
The fraction of the original resolution to scale to. Must be postive.
Note that this can be greater than 1.0, in which case the resolution
is upsampled.
abs_res: list-like
A list-like of the number of pixels for the x and y axes, respectively.
Overrides `frac_res` if specified.
load_params: dict, optional
A dictionary of parameters for `dc.load()`.
Here are some common load parameters:
*lat, lon: list-like 2-tuples of minimum and maximum values for
latitude and longitude, respectively.*
*time: list-like A 2-tuple of the minimum and maximum times
for acquisitions.*
*measurements: list-like The list of measurements to retrieve
from the Datacube.*
masking_params: dict, optional
A dictionary of keyword arguments for corresponding masking functions.
For example: {'cover_types':['cloud']} would retain only clouds for Landsat products,
because `landsat_qa_clean_mask()` is used for the Landsat family of platforms.
indiv_masks: list
A list of masks to return (e.g. ['water']).
These do not have to be the same used to create `clean_mask`.
Returns
-------
dataset: xarray.Dataset
The raw data requested. Can be cleaned with `dataset.where(clean_mask)`.
clean_mask: xarray.DataArray
The clean mask, formed as a logical AND of all masks used.
masks: list of xarray.DataArray
A list of the masks requested by `indiv_masks`,
or `None` if `indiv_masks` is not specified.
Raises
------
AssertionError: If no data is retrieved for any platform query.
"""
from .clean_mask import landsat_qa_clean_mask, landsat_clean_mask_invalid
from .aggregate import xr_scale_res

current_load_params = dict(platform=platform, product=product)
current_load_params.update(load_params)
dataset = dc.load(**current_load_params)
assert len(dataset.dims) > 0, "No data was retrieved."
# Scale resolution if specified.
if frac_res is not None or abs_res is not None:
dataset = xr_scale_res(dataset, frac_res=frac_res, abs_res=abs_res)
# Get the clean mask for the appropriate LANDSAT satellite platform.
clean_mask = landsat_qa_clean_mask(dataset, platform, **masking_params)
# Get the mask for removing data ouside the accepted range of LANDSAT 7 and 8.
clean_mask = xr_and(clean_mask, landsat_clean_mask_invalid(dataset))
# Retrieve individual masks.
if indiv_masks is None:
masks = None
else:
masks = []
for mask in indiv_masks:
masks.append(landsat_qa_clean_mask(dataset, platform, cover_types=[mask]))
return dataset, clean_mask, masks

def load_multiplatform(dc, platforms, products, frac_res=None, abs_res=None,
load_params={}, masking_params={}, indiv_masks=None):
"""
**This function is DEPRECATED.**
Load and merge data as well as clean masks given a list of platforms
and products. Currently only tested on Landsat data.
Parameters
----------
dc: datacube.api.core.Datacube
The Datacube instance to load data with.
platforms, products: list-like
A list-like of platforms and products. Both must have the same length.
frac_res: float
The fraction of the original resolution to scale to. Must be positive.
The x and y dimensions are scaled by the square root of this factor.
Note that this can be greater than 1.0, in which case the resolution
is upsampled. The base resolution used for all products will be the
minimum resolution for latitude and longitude (considered separately -
i.e. one resolution for each dimension) among all of them.
abs_res: list-like
A list-like of the number of pixels for the x and y axes, respectively.
That is, it is a list-like of 2 numbers. Overrides `frac_res` if specified.
load_params: dict, optional
A dictionary of parameters for `dc.load()` or a dictionary of
dictionaries of such parameters, mapping platform names to parameter
dictionaries (primarily useful for selecting different time ranges).
Here are some common load parameters:
*lat, lon: list-like 2-tuples of minimum and maximum values for
latitude and longitude, respectively.*
*time: list-like A pair of the minimum and maximum times
for acquisitions or a list of such pairs.*
*measurements: list-like The list of measurements to retrieve from
the Datacube.*
For example, to load data with different time ranges for different
platforms, we could pass the following:
`{'LANDSAT_7': dict(**common_load_params, time=ls7_date_range),
'LANDSAT_8': dict(**common_load_params, time=ls8_date_range)}`,
where `common_load_params` is a dictionary of load parameters common
to both - most notably 'lat', 'lon', and 'measurements' - and the
'date_range' variables are list-likes of start and end dates.
masking_params: dict, optional
A dictionary mapping platform names to a dictionary of keyword
arguments for corresponding masking functions.
For example: {'LANDSAT_7': {'cover_types':['cloud']},
'LANDSAT_8': {'cover_types': ['cloud']}}
would retain only clouds, because `landsat_qa_clean_mask()` is used
to create clean masks for the Landsat family of platforms.
indiv_masks: list
A list of masks to return (e.g. ['water']). These do not have to be
the same used to create the returned clean mask.
Returns
-------
dataset: xarray.Dataset
The raw data requested. Can be cleaned with `dataset.where(clean_mask)`.
clean_mask: xarray.DataArray
The clean mask, formed as a logical AND of all masks used.
masks: list of xarray.DataArray
A list of the masks requested by `indiv_masks`,
or `None` if `indiv_masks` is not specified.
Raises
------
AssertionError: If no data is retrieved from any product.
"""
# Determine what resolution the data will be scaled to.
if frac_res is not None and abs_res is None:
prod_info = dc.list_products()
resolutions = prod_info[prod_info['name'].isin(products)]\
['resolution'].values
# Determine the minimum resolution, which is actually the maximum
# value resolution, since resolution is measured in degrees per pixel.
# The first resolution is for latitude (y) and is negative.
# The second resolution is for longitude (x) and is positive.
min_res = [0]*2
for res in resolutions:
min_res[0] = res[0] if res[0] < min_res[0] else min_res[0]
min_res[1] = res[1] if min_res[1] < res[1] else min_res[1]
# Take reciprocal to convert degrees per pixel to pixels per degree.
# Reverse to be in order (x, y).
min_res = [abs(frac_res*(1/res)) for res in min_res][::-1]

# Calculate the absolute resolution.
x, y = load_params.get('lon', None), load_params.get('lat', None)
x, y = load_params.get('longitude', x), load_params.get('latitude', y)
x_y_rng = abs(x[1] - x[0]), abs(y[1] - y[0])
abs_res = [round(res*rng) for res, rng in zip(min_res, x_y_rng)]

datasets_temp = {} # Maps platforms to datasets to merge.
clean_masks_temp = {} # Maps platforms to clean masks to merge.
masks_per_platform = {} if indiv_masks is not None else None # Maps platforms to lists of masks.
for product,platform in zip(products, platforms):
current_load_params = dict(platform=platform, product=product)
current_masking_params = masking_params.get(platform, masking_params)

# Handle `load_params` as a dict of dicts of platforms mapping to load params.
if isinstance(list(load_params.values())[0], dict):
current_load_params.update(load_params.get(platform, {}))
else: # Handle `load_params` as a dict of load params.
current_load_params.update(load_params)
# Load each time range of data.
time = current_load_params.get('time')
if isinstance(time[0], tuple) or \
isinstance(time[0], list): # Handle `time` as a list of time ranges.
datasets_time_parts = []
clean_masks_time_parts = []
masks_time_parts = np.empty((len(time), len(indiv_masks)), dtype=object)\
if indiv_masks is not None else None
for i, time_range in enumerate(time):
time_range_load_params = current_load_params
time_range_load_params['time'] = time_range
try:
dataset_time_part, clean_mask_time_part, masks_time_part = \
load_simple(dc, platform, product, abs_res=abs_res,
load_params=time_range_load_params,
masking_params=masking_params,
indiv_masks=indiv_masks)
datasets_time_parts.append(dataset_time_part)
clean_masks_time_parts.append(clean_mask_time_part)
if indiv_masks is not None:
masks_time_parts[i] = masks_time_part
except (AssertionError):
continue
datasets_temp[platform], clean_masks_temp[platform] = \
xarray_concat_and_merge(datasets_time_parts, clean_masks_time_parts)
if indiv_masks is not None:
masks_per_platform[platform] = xarray_concat_and_merge(*masks_time_parts.T)
else: # Handle `time` as a single time range.
try:
datasets_temp[platform], clean_masks_temp[platform], masks = \
load_simple(dc, platform, product, abs_res=abs_res,
load_params=current_load_params,
masking_params=masking_params,
indiv_masks=indiv_masks)
if indiv_masks is not None:
masks_per_platform[platform] = masks
except (AssertionError):
continue
return merge_datasets(datasets_temp, clean_masks_temp, masks_per_platform)

## End Load ##

## Extents ##

def get_product_extents(api, platform, product, **kwargs):
Expand Down

0 comments on commit 9fa8851

Please sign in to comment.