Remove PR old association #47
GitHub Actions / Regression test results for uat
failed
Jul 19, 2024 in 0s
145 fail, 14 skipped, 411 pass in 42m 1s
570 tests 411 ✅ 42m 1s ⏱️
1 suites 14 💤
1 files 145 ❌
Results for commit 65e4807.
Annotations
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1220280430-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 20s]
Raw output
Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
collection_concept_id = 'C1220280430-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1220280430-GES_DISC', 'concept-id': 'G1245884581-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1220280430-GES_DISC'}]}, 'meta': {'association-details': {'collect...L': 'https://cdn.earthdata.nasa.gov/umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'PRODUCT/longitude', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C1220280430')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
> job_id = harmony_client.submit(harmony_request)
verify_collection.py:384:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:851: in submit
self._handle_error_response(response)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <harmony.harmony.Client object at 0x7fb17a3b85b0>
response = <Response [404]>
def _handle_error_response(self, response: Response):
"""Raises the appropriate exception based on the response
received from Harmony. Tries to pull out an error message
from a Harmony JSON response when possible.
Args:
response: The Response from Harmony
Raises:
Exception with a Harmony error message or a more generic
HTTPError
"""
if 'application/json' in response.headers.get('Content-Type', ''):
exception_message = None
try:
response_json = response.json()
if hasattr(response_json, 'get'):
exception_message = response_json.get('description')
if not exception_message:
exception_message = response_json.get('error')
except JSONDecodeError:
pass
if exception_message:
> raise Exception(response.reason, exception_message)
E Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:784: Exception
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1245884581-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1220280430-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-80.170075%3A-62.274925%29&subset=lon%28-71.21815%3A18.87415%29&granuleId=G1245884581-GES_DISC
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1220280440-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 13s]
Raw output
Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
collection_concept_id = 'C1220280440-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1220280440-GES_DISC', 'concept-id': 'G1256524037-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1220280440-GES_DISC'}]}, 'meta': {'association-details': {'collect.../umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'METADATA/QA_STATISTICS/aerosol_mid_pressure_pdf_axis', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C1220280440')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
> job_id = harmony_client.submit(harmony_request)
verify_collection.py:384:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:851: in submit
self._handle_error_response(response)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <harmony.harmony.Client object at 0x7fb17737f4f0>
response = <Response [404]>
def _handle_error_response(self, response: Response):
"""Raises the appropriate exception based on the response
received from Harmony. Tries to pull out an error message
from a Harmony JSON response when possible.
Args:
response: The Response from Harmony
Raises:
Exception with a Harmony error message or a more generic
HTTPError
"""
if 'application/json' in response.headers.get('Content-Type', ''):
exception_message = None
try:
response_json = response.json()
if hasattr(response_json, 'get'):
exception_message = response_json.get('description')
if not exception_message:
exception_message = response_json.get('error')
except JSONDecodeError:
pass
if exception_message:
> raise Exception(response.reason, exception_message)
E Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:784: Exception
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1256524037-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1220280440-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-72.00995%3A-56.276050000000005%29&subset=lon%28-153.63479999999998%3A140.6828%29&granuleId=G1256524037-GES_DISC
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1236469823-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 1m 26s]
Raw output
OSError: [Errno group not found: PRODUCT] 'PRODUCT'
ds = <[AttributeError('NetCDF: Not a valid ID') raised in repr()] Group object at 0x7f01d35f0d40>
group = '/METADATA/PRODUCT', mode = 'r'
create_group = <function _netcdf4_create_group at 0x7f01d7b205e0>
def _nc4_require_group(ds, group, mode, create_group=_netcdf4_create_group):
if group in {None, "", "/"}:
# use the root group
return ds
else:
# make sure it's a string
if not isinstance(group, str):
raise ValueError("group must be a string or None")
# support path-like syntax
path = group.strip("/").split("/")
for key in path:
try:
> ds = ds.groups[key]
E KeyError: 'PRODUCT'
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:190: KeyError
During handling of the above exception, another exception occurred:
collection_concept_id = 'C1236469823-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1236469823-GES_DISC', 'concept-id': 'G1262646684-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1236469823-GES_DISC'}]}, 'meta': {'association-details': {'collect...able/v1.9.0', 'Version': '1.9.0'}, 'Name': 'PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/fitted_radiance_squeeze', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw3/test_spatial_subset_C1236469820')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
lat_var_name = lat_var_name.split('/')[-1]
lon_var_name = lon_var_name.split('/')[-1]
with netCDF4.Dataset(subsetted_filepath) as f:
group_list = []
def group_walk(groups, nc_d, current_group):
global subsetted_ds_new
subsetted_ds_new = None
# check if the top group has lat or lon variable
if lat_var_name in list(nc_d.variables.keys()):
subsetted_ds_new = subsetted_ds
else:
# if not then we'll need to keep track of the group layers
group_list.append(current_group)
# loop through the groups in the current layer
for g in groups:
# end the loop if we've already found latitude
if subsetted_ds_new:
break
# check if the groups have latitude, define the dataset and end the loop if found
if lat_var_name in list(nc_d.groups[g].variables.keys()):
group_list.append(g)
lat_group = '/'.join(group_list)
subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=lat_group, decode_times=False)
# add a science variable to the dataset if other groups are in the lat/lon group
# some GPM collections won't have any other variables in the same group as lat/lon
if len(list(nc_d.groups[g].groups.keys())) > 0:
data_group = [v for v in list(nc_d.groups[g].groups.keys()) if 'time' not in str(v).lower()][0]
g_data = lat_group+'/'+data_group
subsetted_ds_data = xarray.open_dataset(subsetted_filepath, group=g_data, decode_times=False)
sci_var = list(subsetted_ds_data.variables.keys())[0]
subsetted_ds_new['science_test'] = subsetted_ds_data[sci_var]
break
# recall the function on a group that has groups in it and didn't find latitude
# this is going 'deeper' into the groups
if len(list(nc_d.groups[g].groups.keys())) > 0:
group_walk(nc_d.groups[g].groups, nc_d.groups[g], g)
else:
continue
> group_walk(f.groups, f, '')
verify_collection.py:440:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:423: in group_walk
subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=lat_group, decode_times=False)
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/api.py:571: in open_dataset
backend_ds = backend.open_dataset(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:646: in open_dataset
store = NetCDF4DataStore.open(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:409: in open
return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:356: in __init__
self.format = self.ds.data_model
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:418: in ds
return self._acquire()
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:413: in _acquire
ds = _nc4_require_group(root, self._group, self._mode)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
ds = <[AttributeError('NetCDF: Not a valid ID') raised in repr()] Group object at 0x7f01d35f0d40>
group = '/METADATA/PRODUCT', mode = 'r'
create_group = <function _netcdf4_create_group at 0x7f01d7b205e0>
def _nc4_require_group(ds, group, mode, create_group=_netcdf4_create_group):
if group in {None, "", "/"}:
# use the root group
return ds
else:
# make sure it's a string
if not isinstance(group, str):
raise ValueError("group must be a string or None")
# support path-like syntax
path = group.strip("/").split("/")
for key in path:
try:
ds = ds.groups[key]
except KeyError as e:
if mode != "r":
ds = create_group(ds, key)
else:
# wrap error to provide slightly more helpful message
> raise OSError(f"group not found: {key}", e)
E OSError: [Errno group not found: PRODUCT] 'PRODUCT'
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:196: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1262646684-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1236469823-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-77.06075%3A-59.913250000000005%29&subset=lon%2873.40985%3A149.86015%29&granuleId=G1262646684-GES_DISC
INFO root:verify_collection.py:385 Submitted harmony job 7fdfa133-681e-4767-a4bc-80374b49b425
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw3/test_spatial_subset_C1236469820/4398865_S5P_OFFL_L2_SO2_20240223T162229_20240223T180359_32976_03_020601_20240225T143938_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1229246436-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 30s]
Raw output
Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
collection_concept_id = 'C1229246436-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1229246436-GES_DISC', 'concept-id': 'G1256524046-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1229246436-GES_DISC'}]}, 'meta': {'association-details': {'collect...v1.9.0', 'Version': '1.9.0'}, 'Name': 'PRODUCT/SUPPORT_DATA/DETAILED_RESULTS/fitted_radiance_squeeze_win2', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1229246430')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
> job_id = harmony_client.submit(harmony_request)
verify_collection.py:384:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:851: in submit
self._handle_error_response(response)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <harmony.harmony.Client object at 0x7f37031327d0>
response = <Response [404]>
def _handle_error_response(self, response: Response):
"""Raises the appropriate exception based on the response
received from Harmony. Tries to pull out an error message
from a Harmony JSON response when possible.
Args:
response: The Response from Harmony
Raises:
Exception with a Harmony error message or a more generic
HTTPError
"""
if 'application/json' in response.headers.get('Content-Type', ''):
exception_message = None
try:
response_json = response.json()
if hasattr(response_json, 'get'):
exception_message = response_json.get('description')
if not exception_message:
exception_message = response_json.get('error')
except JSONDecodeError:
pass
if exception_message:
> raise Exception(response.reason, exception_message)
E Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:784: Exception
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1256524046-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1229246436-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-63.83435%3A-48.571650000000005%29&subset=lon%28-144.50625%3A159.06625%29&granuleId=G1256524046-GES_DISC
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1236469820-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 32s]
Raw output
Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
collection_concept_id = 'C1236469820-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1236469820-GES_DISC', 'concept-id': 'G1262646723-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1236469820-GES_DISC'}]}, 'meta': {'association-details': {'collect...tracted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': 9.969209968386869e+36}], ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw2/test_spatial_subset_C1236469820')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
> job_id = harmony_client.submit(harmony_request)
verify_collection.py:384:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:851: in submit
self._handle_error_response(response)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <harmony.harmony.Client object at 0x7f5a6979f880>
response = <Response [404]>
def _handle_error_response(self, response: Response):
"""Raises the appropriate exception based on the response
received from Harmony. Tries to pull out an error message
from a Harmony JSON response when possible.
Args:
response: The Response from Harmony
Raises:
Exception with a Harmony error message or a more generic
HTTPError
"""
if 'application/json' in response.headers.get('Content-Type', ''):
exception_message = None
try:
response_json = response.json()
if hasattr(response_json, 'get'):
exception_message = response_json.get('description')
if not exception_message:
exception_message = response_json.get('error')
except JSONDecodeError:
pass
if exception_message:
> raise Exception(response.reason, exception_message)
E Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:784: Exception
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1262646723-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1236469820-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-77.16915%3A-59.99885%29&subset=lon%28-28.32865%3A48.52065%29&granuleId=G1262646723-GES_DISC
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1240921713-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 27s]
Raw output
Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
collection_concept_id = 'C1240921713-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1240921713-GES_DISC', 'concept-id': 'G1262637717-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1240921713-GES_DISC'}]}, 'meta': {'association-details': {'collect..., 'Version': '1.9.0'}, 'Name': 'METADATA/QA_STATISTICS/nitrogendioxide_tropospheric_column_histogram_axis', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw1/test_spatial_subset_C1240921710')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
> job_id = harmony_client.submit(harmony_request)
verify_collection.py:384:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:851: in submit
self._handle_error_response(response)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <harmony.harmony.Client object at 0x7f6285a37310>
response = <Response [404]>
def _handle_error_response(self, response: Response):
"""Raises the appropriate exception based on the response
received from Harmony. Tries to pull out an error message
from a Harmony JSON response when possible.
Args:
response: The Response from Harmony
Raises:
Exception with a Harmony error message or a more generic
HTTPError
"""
if 'application/json' in response.headers.get('Content-Type', ''):
exception_message = None
try:
response_json = response.json()
if hasattr(response_json, 'get'):
exception_message = response_json.get('description')
if not exception_message:
exception_message = response_json.get('error')
except JSONDecodeError:
pass
if exception_message:
> raise Exception(response.reason, exception_message)
E Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:784: Exception
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1262637717-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1240921713-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-64.344925%3A-49.028075%29&subset=lon%288.573025000000001%3A56.757975%29&granuleId=G1262637717-GES_DISC
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1220280433-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 20s]
Raw output
Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
collection_concept_id = 'C1220280433-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1220280433-GES_DISC', 'concept-id': 'G1256524039-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1220280433-GES_DISC'}]}, 'meta': {'association-details': {'collect...ov/umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'METADATA/QA_STATISTICS/nitrogendioxide_total_column_pdf'}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw1/test_spatial_subset_C1220280430')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
> job_id = harmony_client.submit(harmony_request)
verify_collection.py:384:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:851: in submit
self._handle_error_response(response)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <harmony.harmony.Client object at 0x7f6284d7a0b0>
response = <Response [404]>
def _handle_error_response(self, response: Response):
"""Raises the appropriate exception based on the response
received from Harmony. Tries to pull out an error message
from a Harmony JSON response when possible.
Args:
response: The Response from Harmony
Raises:
Exception with a Harmony error message or a more generic
HTTPError
"""
if 'application/json' in response.headers.get('Content-Type', ''):
exception_message = None
try:
response_json = response.json()
if hasattr(response_json, 'get'):
exception_message = response_json.get('description')
if not exception_message:
exception_message = response_json.get('error')
except JSONDecodeError:
pass
if exception_message:
> raise Exception(response.reason, exception_message)
E Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:784: Exception
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1256524039-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1220280433-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-72.68005%3A-56.265950000000004%29&subset=lon%28-171.89765%3A-108.42435%29&granuleId=G1256524039-GES_DISC
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1262899917-LARC_CLOUD] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 37s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1262899917-LARC_CLOUD', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1262899917-LARC_CLOUD', 'concept-id': 'G1263137356-LARC_CLOUD', 'concept-type': '...ProductionDateTime': '2024-04-23T15:39:34+00:00'}, 'GranuleUR': 'TEMPO_O3TOT_L2_V03_20240328T163657Z_S008G09.nc', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1262899917-LARC_CLOUD'}]}, 'meta': {'association-details': {'colle...acted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -1.0000000150474662e+30}], ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw5/test_spatial_subset_C1262899910')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1263137356-LARC_CLOUD for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1262899917-LARC_CLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%2818.491264724975%3A58.17076835602501%29&subset=lon%28-148.0711017608%3A-114.2712596892%29&granuleId=G1263137356-LARC_CLOUD
INFO root:verify_collection.py:385 Submitted harmony job e56fcd14-3c7a-4049-9d90-9fa939fe89cd
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw5/test_spatial_subset_C1262899910/4398993_TEMPO_O3TOT_L2_V03_20240328T163657Z_S008G09_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1236469821-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 15s]
Raw output
Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
collection_concept_id = 'C1236469821-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1236469821-GES_DISC', 'concept-id': 'G1262646629-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1236469821-GES_DISC'}]}, 'meta': {'association-details': {'collect...L': 'https://cdn.earthdata.nasa.gov/umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'PRODUCT/longitude', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_spatial_subset_C1236469820')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
> job_id = harmony_client.submit(harmony_request)
verify_collection.py:384:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:851: in submit
self._handle_error_response(response)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <harmony.harmony.Client object at 0x7f72a10f9660>
response = <Response [404]>
def _handle_error_response(self, response: Response):
"""Raises the appropriate exception based on the response
received from Harmony. Tries to pull out an error message
from a Harmony JSON response when possible.
Args:
response: The Response from Harmony
Raises:
Exception with a Harmony error message or a more generic
HTTPError
"""
if 'application/json' in response.headers.get('Content-Type', ''):
exception_message = None
try:
response_json = response.json()
if hasattr(response_json, 'get'):
exception_message = response_json.get('description')
if not exception_message:
exception_message = response_json.get('error')
except JSONDecodeError:
pass
if exception_message:
> raise Exception(response.reason, exception_message)
E Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:784: Exception
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1262646629-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1236469821-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-77.06075%3A-59.913250000000005%29&subset=lon%2873.40985%3A149.86015%29&granuleId=G1262646629-GES_DISC
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1261072658-POCLOUD] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 1m 9s]
Raw output
AttributeError: NetCDF: Can't open HDF5 attribute
collection_concept_id = 'C1261072658-POCLOUD', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1261072658-POCLOUD', 'concept-id': 'G1261339526-POCLOUD', 'concept-type': 'granul...DateTime': '2023-11-21T23:39:05.675Z'}, 'GranuleUR': 'SWOT_OPN_2PfS006_540_20231121_195406_20231121_214049_swot', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1261072658-POCLOUD'}]}, 'meta': {'association-details': {'collecti...1 Hz altimeter data: C band SWH', 'Dimensions': [{'Name': 'time', 'Size': 5558, 'Type': 'TIME_DIMENSION'}], ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C1261072650')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:328: in get_lat_lon_var_names
with xarray.open_dataset(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/api.py:571: in open_dataset
backend_ds = backend.open_dataset(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/store.py:43: in open_dataset
vars, attrs = filename_or_obj.load()
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/common.py:253: in load
attributes = FrozenDict(self.get_attrs())
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:467: in get_attrs
return FrozenDict((k, self.ds.getncattr(k)) for k in self.ds.ncattrs())
src/netCDF4/_netCDF4.pyx:3035: in netCDF4._netCDF4.Dataset.ncattrs
???
src/netCDF4/_netCDF4.pyx:1521: in netCDF4._netCDF4._get_att_names
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E AttributeError: NetCDF: Can't open HDF5 attribute
src/netCDF4/_netCDF4.pyx:2034: AttributeError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1261339526-POCLOUD for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1261072658-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-75.66962749999999%3A0.4357274999999987%29&subset=lon%2845.80655750000001%3A176.5591425%29&granuleId=G1261339526-POCLOUD
INFO root:verify_collection.py:385 Submitted harmony job 45e692fe-3999-4e1d-b869-196b00aa5097
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C1261072650/4398995_SWOT_OPN_2PfS006_540_20231121_195406_20231121_214049_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1240560262-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 2m 0s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1240560262-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1240560262-GES_DISC', 'concept-id': 'G1243148634-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1240560262-GES_DISC'}]}, 'meta': {'association-details': {'collect...': 'Extracted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -9999.900390625}], ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C1240560260')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1243148634-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1240560262-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-65.64187875%3A-61.05637125%29&subset=lon%28-139.720288%3A-110.199532%29&granuleId=G1243148634-GES_DISC
INFO root:verify_collection.py:385 Submitted harmony job 50654b08-5902-4945-9e28-faf6a8625d50
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C1240560260/4399003_2A.GPM.Ka.V9-20211125.20220102-S223824-E001057.044589.V07A_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1240921715-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 17s]
Raw output
Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
collection_concept_id = 'C1240921715-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1240921715-GES_DISC', 'concept-id': 'G1262646682-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1240921715-GES_DISC'}]}, 'meta': {'association-details': {'collect....nasa.gov/umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'PRODUCT/methane_mixing_ratio_bias_corrected', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw3/test_spatial_subset_C1240921710')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
> job_id = harmony_client.submit(harmony_request)
verify_collection.py:384:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:851: in submit
self._handle_error_response(response)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <harmony.harmony.Client object at 0x7f01d32d28c0>
response = <Response [404]>
def _handle_error_response(self, response: Response):
"""Raises the appropriate exception based on the response
received from Harmony. Tries to pull out an error message
from a Harmony JSON response when possible.
Args:
response: The Response from Harmony
Raises:
Exception with a Harmony error message or a more generic
HTTPError
"""
if 'application/json' in response.headers.get('Content-Type', ''):
exception_message = None
try:
response_json = response.json()
if hasattr(response_json, 'get'):
exception_message = response_json.get('description')
if not exception_message:
exception_message = response_json.get('error')
except JSONDecodeError:
pass
if exception_message:
> raise Exception(response.reason, exception_message)
E Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:784: Exception
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1262646682-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1240921715-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-77.02635%3A-59.977650000000004%29&subset=lon%28-2.3436499999999967%3A72.87164999999999%29&granuleId=G1262646682-GES_DISC
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1215720327-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 22s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1215720327-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1215720327-GES_DISC', 'concept-id': 'G1265717607-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1215720327-GES_DISC'}]}, 'meta': {'association-details': {'collect... 'Value': -1e+30}], 'LongName': 'HDFEOS/SWATHS/OMI Slant Column Amount OClO/Data Fields/AverageFittingRMS', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw5/test_spatial_subset_C1215720320')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1265717607-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1215720327-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-73.54333425%3A-50.597575750000004%29&subset=lon%28-89.68945275%3A-19.574617250000003%29&granuleId=G1265717607-GES_DISC
INFO root:verify_collection.py:385 Submitted harmony job cf46e962-9067-4621-82d3-43a366a9c6b8
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw5/test_spatial_subset_C1215720320/4399010_OMI-Aura_L2-OMOCLO_2023m1225t0356-o103423_v003-2023m1225t102049_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1234666469-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 26s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1234666469-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1234666469-GES_DISC', 'concept-id': 'G1256346630-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1234666469-GES_DISC'}]}, 'meta': {'association-details': {'collect...'SCIENCE_FILLVALUE', 'Value': -999}], 'LongName': 'HDFEOS/SWATHS/O3 column/Geolocation Fields/ChunkNumber', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1234666460')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1256346630-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1234666469-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-85.5%3A85.5%29&subset=lon%28-171.0%3A171.0%29&granuleId=G1256346630-GES_DISC
INFO root:verify_collection.py:385 Submitted harmony job c2a5c96e-2a5e-47c4-8ad9-525721ad88ba
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1234666460/4399011_MLS-Aura_L2GP-O3_v05-01-c01_2022d318_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1229246435-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 18s]
Raw output
Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
collection_concept_id = 'C1229246435-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1229246435-GES_DISC', 'concept-id': 'G1256523832-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1229246435-GES_DISC'}]}, 'meta': {'association-details': {'collect...L': 'https://cdn.earthdata.nasa.gov/umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'PRODUCT/longitude', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_spatial_subset_C1229246430')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
> job_id = harmony_client.submit(harmony_request)
verify_collection.py:384:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:851: in submit
self._handle_error_response(response)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <harmony.harmony.Client object at 0x7f729bd2f520>
response = <Response [404]>
def _handle_error_response(self, response: Response):
"""Raises the appropriate exception based on the response
received from Harmony. Tries to pull out an error message
from a Harmony JSON response when possible.
Args:
response: The Response from Harmony
Raises:
Exception with a Harmony error message or a more generic
HTTPError
"""
if 'application/json' in response.headers.get('Content-Type', ''):
exception_message = None
try:
response_json = response.json()
if hasattr(response_json, 'get'):
exception_message = response_json.get('description')
if not exception_message:
exception_message = response_json.get('error')
except JSONDecodeError:
pass
if exception_message:
> raise Exception(response.reason, exception_message)
E Exception: ('Not Found', 'Error: EULA 2f16e13a-e648-4920-9276-d578f30707c6 could not be found.')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:784: Exception
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1256523832-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1229246435-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-63.821374999999996%3A-48.559625000000004%29&subset=lon%28-73.420925%3A-25.956075000000002%29&granuleId=G1256523832-GES_DISC
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1261072662-POCLOUD] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 39s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1261072662-POCLOUD', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1261072662-POCLOUD', 'concept-id': 'G1261601012-POCLOUD', 'concept-type': 'granul...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1261072662-POCLOUD'}]}, 'meta': {'association-details': {'collecti..., 'Size': 56663, 'Type': 'TIME_DIMENSION'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': 32767}], ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw9/test_spatial_subset_C1261072660')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1261601012-POCLOUD for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1261072662-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-73.87973249999999%3A70.2720325%29&subset=lon%28-66.1692675%3A173.6879675%29&granuleId=G1261601012-POCLOUD
INFO root:verify_collection.py:385 Submitted harmony job f6cfa8ab-49c6-49cc-a83b-94df260262d6
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw9/test_spatial_subset_C1261072660/4399013_SWOT_OPRAD_2PaS009_212_20240111_163300_20240111_181913_PIC0_01_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1215720121-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 24s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1215720121-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1215720121-GES_DISC', 'concept-id': 'G1257041856-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1215720121-GES_DISC'}]}, 'meta': {'association-details': {'collect...'SCIENCE_FILLVALUE', 'Value': -32767}], 'LongName': 'HDFEOS/SWATHS/O3Profile/Data Fields/CovarianceMatrix', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_spatial_subset_C1215720120')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1257041856-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1215720121-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-75.0429%3A-52.46368%29&subset=lon%28101.57137925%3A173.93209075%29&granuleId=G1257041856-GES_DISC
INFO root:verify_collection.py:385 Submitted harmony job d6b9d81b-06c4-40a8-88cb-47814dbc9336
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw0/test_spatial_subset_C1215720120/4399018_OMI-Aura_L2-OMO3PR_2019m1203t1459-o81832_v003-2019m1205t075950_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1215720126-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 28s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1215720126-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1215720126-GES_DISC', 'concept-id': 'G1244933262-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1215720126-GES_DISC'}]}, 'meta': {'association-details': {'collect.../umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'HDFEOS/SWATHS/ColumnAmountNO2/Data Fields/AmfTropStd', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw5/test_spatial_subset_C1215720120')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1244933262-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1215720126-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-75.44479475%3A-52.37251525%29&subset=lon%28-34.797884249999996%3A42.03947425%29&granuleId=G1244933262-GES_DISC
INFO root:verify_collection.py:385 Submitted harmony job acd7b774-5ea9-4c12-b064-b3bf82d22476
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw5/test_spatial_subset_C1215720120/4399023_OMI-Aura_L2-OMNO2_2022m0110t2357-o93036_v003-2022m0331t001503_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1256783382-POCLOUD] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 1m 30s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1256783382-POCLOUD', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1256783382-POCLOUD', 'concept-id': 'G1257897007-POCLOUD', 'concept-type': 'granul...DateTime': '2023-05-15T11:13:58.868Z'}, 'GranuleUR': 'SWOT_IPS_2PfP520_027_20230514_125933_20230514_135039_swot', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1256783382-POCLOUD'}]}, 'meta': {'association-details': {'collecti...', 'Size': 2885, 'Type': 'TIME_DIMENSION'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': 32767}], ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw2/test_spatial_subset_C1256783380')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1257897007-POCLOUD for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1256783382-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-85.5%3A85.5%29&subset=lon%28-171.0%3A171.0%29&granuleId=G1257897007-POCLOUD
INFO root:verify_collection.py:385 Submitted harmony job eedbab0d-bc76-4322-a5f7-a17f875aa3bf
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw2/test_spatial_subset_C1256783380/4399027_SWOT_IPS_2PfP520_027_20230514_125933_20230514_135039_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1215720125-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 32s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1215720125-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1215720125-GES_DISC', 'concept-id': 'G1265717650-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1215720125-GES_DISC'}]}, 'meta': {'association-details': {'collect...acted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -1.0000000150474662e+30}], ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C1215720120')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1265717650-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1215720125-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-73.62743875%3A-50.79357125%29&subset=lon%28-15.377333249999992%3A54.17926325%29&granuleId=G1265717650-GES_DISC
INFO root:verify_collection.py:385 Submitted harmony job 2d83ce2d-b624-479d-b75b-452dacac7ed8
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C1215720120/4399035_OMI-Aura_L2-OMHCHO_2023m1224t2300-o103420_v003-2023m1225t083641_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1215720123-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 25s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1215720123-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1215720123-GES_DISC', 'concept-id': 'G1265717636-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1215720123-GES_DISC'}]}, 'meta': {'association-details': {'collect...escription': 'Extracted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -9999}], ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw3/test_spatial_subset_C1215720120')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1265717636-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1215720123-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-73.61424724999999%3A-50.78146275%29&subset=lon%28-89.46278225%3A-19.97036775%29&granuleId=G1265717636-GES_DISC
INFO root:verify_collection.py:385 Submitted harmony job df11d6b3-6d61-4992-bf55-ce597e371555
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw3/test_spatial_subset_C1215720120/4399039_OMI-Aura_L2-OMCLDRR_2023m1225t0356-o103423_v003-2023m1225t093618_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1229246430-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 38s]
Raw output
AttributeError: NetCDF: Can't open HDF5 attribute
collection_concept_id = 'C1229246430-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1229246430-GES_DISC', 'concept-id': 'G1256244974-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1229246430-GES_DISC'}]}, 'meta': {'association-details': {'collect...gov/umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'METADATA/QA_STATISTICS/aerosol_index_354_388_histogram'}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C1229246430')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:328: in get_lat_lon_var_names
with xarray.open_dataset(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/api.py:571: in open_dataset
backend_ds = backend.open_dataset(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/store.py:43: in open_dataset
vars, attrs = filename_or_obj.load()
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/common.py:253: in load
attributes = FrozenDict(self.get_attrs())
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:467: in get_attrs
return FrozenDict((k, self.ds.getncattr(k)) for k in self.ds.ncattrs())
src/netCDF4/_netCDF4.pyx:3035: in netCDF4._netCDF4.Dataset.ncattrs
???
src/netCDF4/_netCDF4.pyx:1521: in netCDF4._netCDF4._get_att_names
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E AttributeError: NetCDF: Can't open HDF5 attribute
src/netCDF4/_netCDF4.pyx:2034: AttributeError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1256244974-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1229246430-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-76.865725%3A-59.755275%29&subset=lon%28-153.39715%3A-77.70685%29&granuleId=G1256244974-GES_DISC
INFO root:verify_collection.py:385 Submitted harmony job c2ca5c4b-be69-4109-b100-c8940e2f5ecc
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C1229246430/4399043_S5P_OFFL_L2_AER_AI_20210701T220753_20210701T234923_19260_02_020200_20210707T075659_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1261072650-POCLOUD] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 1m 18s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1261072650-POCLOUD', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1261072650-POCLOUD', 'concept-id': 'G1261713031-POCLOUD', 'concept-type': 'granul...DateTime': '2023-12-22T06:55:23.367Z'}, 'GranuleUR': 'SWOT_GPN_2PfP578_003_20230710_072954_20230710_082100_swot', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1261072650-POCLOUD'}]}, 'meta': {'association-details': {'collecti...ize': 2278, 'Type': 'TIME_DIMENSION'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': 2147483647}], ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw9/test_spatial_subset_C1261072650')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1261713031-POCLOUD for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1261072650-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-74.475425%3A47.135024999999985%29&subset=lon%28-84.44192%3A5.932719999999996%29&granuleId=G1261713031-POCLOUD
INFO root:verify_collection.py:385 Submitted harmony job eae48f87-6925-4af5-b068-8192d4e08000
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw9/test_spatial_subset_C1261072650/4399047_SWOT_GPN_2PfP578_003_20230710_072954_20230710_082100_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1261072651-POCLOUD] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 1m 35s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1261072651-POCLOUD', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1261072651-POCLOUD', 'concept-id': 'G1261703070-POCLOUD', 'concept-type': 'granul...DateTime': '2023-12-22T06:55:23.413Z'}, 'GranuleUR': 'SWOT_GPS_2PfP578_003_20230710_072954_20230710_082100_swot', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1261072651-POCLOUD'}]}, 'meta': {'association-details': {'collecti...r ocean; 1 = less than 4 points', 'Dimensions': [{'Name': 'time', 'Size': 2278, 'Type': 'TIME_DIMENSION'}], ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw0/test_spatial_subset_C1261072650')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1261703070-POCLOUD for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1261072651-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-74.475425%3A47.135024999999985%29&subset=lon%28-84.44192%3A5.932719999999996%29&granuleId=G1261703070-POCLOUD
INFO root:verify_collection.py:385 Submitted harmony job 25fdeb83-f01f-428c-a87d-ce4bc86b336e
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw0/test_spatial_subset_C1261072650/4399051_SWOT_GPS_2PfP578_003_20230710_072954_20230710_082100_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Check warning on line 0 in tests.verify_collection
github-actions / Regression test results for uat
test_spatial_subset[C1234666467-GES_DISC] (tests.verify_collection) failed
test-results/uat_test_report.xml [took 34s]
Raw output
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1234666467-GES_DISC', env = 'uat'
granule_json = {'meta': {'collection-concept-id': 'C1234666467-GES_DISC', 'concept-id': 'G1256346685-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1234666467-GES_DISC'}]}, 'meta': {'association-details': {'collect...E', 'Value': -999.989990234375}], 'LongName': 'HDFEOS/SWATHS/HNO3-190/Geolocation Fields/LineOfSightAngle', ...}}, ...]
harmony_env = <Environment.UAT: 3>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C1234666460')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfdWF0IiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...JsVVg8CqwJbKrPtsp8q_IpTvyoiKs384GlDUAXyucjzV45fZ8QogcmZHuNfH_2D_zjR7lLCMFV1J82hpJoUH0wlnUYq95g9cxjzPpnRLUaERRyD193TdyA'
@pytest.mark.timeout(600)
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)
# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
> lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
verify_collection.py:398:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
verify_collection.py:318: in get_lat_lon_var_names
nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO root:verify_collection.py:365 Using granule G1256346685-GES_DISC for test
INFO root:verify_collection.py:381 Sending harmony request https://harmony.uat.earthdata.nasa.gov/C1234666467-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-85.5%3A85.5%29&subset=lon%28-171.0%3A171.0%29&granuleId=G1256346685-GES_DISC
INFO root:verify_collection.py:385 Submitted harmony job d091c119-0392-4ba5-8087-aad6632733d9
INFO root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C1234666460/4399055_MLS-Aura_L2GP-HNO3_v05-01-c01_2022d318_subsetted.nc4
WARNING root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
Loading