From e051962a4efffdb1b13b10cb0de9da8a0088a591 Mon Sep 17 00:00:00 2001 From: vggonzal <9Tcostoamm> Date: Mon, 18 Sep 2023 15:08:13 -0700 Subject: [PATCH] added draft of lakes endpoints --- hydrocronapi/controllers/lakessubset.py | 231 ++++++++++++++++++++ hydrocronapi/controllers/lakestimeseries.py | 215 ++++++++++++++++++ hydrocronapi/swagger/swagger.yaml | 205 ++++++++++++++++- 3 files changed, 649 insertions(+), 2 deletions(-) create mode 100644 hydrocronapi/controllers/lakessubset.py create mode 100644 hydrocronapi/controllers/lakestimeseries.py diff --git a/hydrocronapi/controllers/lakessubset.py b/hydrocronapi/controllers/lakessubset.py new file mode 100644 index 0000000..e8f7c7c --- /dev/null +++ b/hydrocronapi/controllers/lakessubset.py @@ -0,0 +1,231 @@ +# pylint: disable=duplicate-code +# pylint: disable=R1702 +# pylint: disable=W0613 +# pylint: disable=E0401 +# pylint: disable=R0912 +# pylint: disable=R0915 +"""Module defining Lambda workflow for subset endpoint.""" + +import json +import logging +import time +from datetime import datetime +from typing import Generator + +from shapely import Polygon, Point + +from hydrocronapi.controllers.db import db + +logger = logging.getLogger() + + +def getlakessubset_get(feature, subsetpolygon, start_time, end_time, output, fields): # noqa: E501 + """Subset by time series for a given spatial region + + Get Timeseries for a particular Reach, Node, or LakeID # noqa: E501 + + :param start_time: Start time of the timeseries + :type start_time: str + :param end_time: End time of the timeseries + :type end_time: str + :param subsetpolygon: GEOJSON of the subset area + :type subsetpolygon: str + :param format: Format of the data returned + :type format: str + + :rtype: None + """ + + polygon = Polygon(json.loads(subsetpolygon)['features'][0]['geometry']['coordinates']) + + start_time = start_time.replace("T", " ")[0:19] + end_time = end_time.replace("T", " ")[0:19] + start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S") + end_time = datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S") + + start = time.time() + results = db.get_series(feature.lower(), start_time, end_time) + + end = time.time() + + data = "" + if output == 'geojson': + data = format_subset_json(results, polygon, start_time, end_time, round((end - start) * 1000, 3)) + if output == 'csv': + data = format_subset_csv(results, polygon, fields, start_time, end_time, round((end - start) * 1000, 3)) + + return data + + +def format_subset_json(results: Generator, polygon, start_time, end_time, elapsed_time): + """ + + Parameters + ---------- + results + polygon + elapsed_time + + Returns + ------- + data + """ + # Fetch all results from query + if 'Items' in results: + results = results['Items'] + else: + results = [results['Item']] + + data = {} + + if results is None: + data['error'] = f"404: Results with the specified polygon {polygon} were not found." + elif len(results) > 5750000: + data['error'] = f'413: Query exceeds 6MB with {len(results)} hits.' + else: + data['status'] = "200 OK" + data['time'] = str(elapsed_time) + " ms." + # data['search on'] = {"feature_id": feature_id} + data['type'] = "FeatureCollection" + data['features'] = [] + i = 0 + for res in results: + if res['time'] != '-999999999999': # and (res['width'] != '-999999999999')): + time_str = datetime.strptime(res['time_str'].replace("T", " ")[0:19], "%Y-%m-%d %H:%M:%S") + if start_time <= time_str <= end_time: + feature = {} + feature['properties'] = {} + feature['geometry'] = {} + feature['type'] = "Feature" + feature['geometry']['coordinates'] = [] + + point = Point(float(res['p_lon']), float(res['p_lat'])) + if polygon.contains(point): + feature_type = '' + if 'POINT' in res['geometry']: + geometry = res['geometry'].replace('POINT (', '').replace(')', '') + geometry = geometry.replace('"', '') + geometry = geometry.replace("'", "") + feature_type = 'Point' + if 'LINESTRING' in res['geometry']: + geometry = res['geometry'].replace('LINESTRING (', '').replace(')', '') + geometry = geometry.replace('"', '') + geometry = geometry.replace("'", "") + feature_type = 'LineString' + + feature['geometry']['type'] = feature_type + if feature_type == 'LineString': + for pol in geometry.split(", "): + (var_x, var_y) = pol.split(" ") + feature['geometry']['coordinates'].append([float(var_x), float(var_y)]) + feature['properties']['time'] = datetime.fromtimestamp( + float(res['time']) + 946710000).strftime("%Y-%m-%d %H:%M:%S") + feature['properties']['reach_id'] = float(res['reach_id']) + feature['properties']['wse'] = float(res['wse']) + + if feature_type == 'Point': + feature['geometry']['coordinates'] = [float(res['p_lon']), float(res['p_lat'])] + feature['properties']['time'] = datetime.fromtimestamp(float(res['time']) + 946710000).strftime( + "%Y-%m-%d %H:%M:%S") + feature['properties']['reach_id'] = float(res['reach_id']) + feature['properties']['wse'] = float(res['wse']) + + data['features'].append(feature) + i += 1 + data['hits'] = i + print(data) + return data + + +def format_subset_csv(results: Generator, polygon, fields, start_time, end_time, elapsed_time): + """ + + Parameters + ---------- + results + polygon + fields + elapsed_time + + Returns + ------- + data + """ + # Fetch all results from query + if 'Items' in results: + results = results['Items'] + else: + results = [results['Item']] + + data = {} + + if results is None: + data['error'] = f"404: Results with the specified polygon {polygon} were not found." + elif len(results) > 5750000: + data['error'] = f'413: Query exceeds 6MB with {len(results)} hits.' + + else: + data['status'] = "200 OK" + data['time'] = str(elapsed_time) + " ms." + # data['search on'] = {"feature_id": feature_id} + data['type'] = "FeatureCollection" + data['features'] = [] + i = 0 + csv = fields + '\n' + fields_set = fields.split(", ") + for res in results: + if res['time'] != '-999999999999': # and (res['width'] != '-999999999999')): + time_str = datetime.strptime(res['time_str'].replace("T", " ")[0:19], "%Y-%m-%d %H:%M:%S") + if start_time <= time_str <= end_time: + point = Point(float(res['p_lon']), float(res['p_lat'])) + if polygon.contains(point): + if 'reach_id' in fields_set: + csv += res['reach_id'] + csv += ',' + if 'time_str' in fields_set: + csv += res['time_str'] + csv += ',' + if 'wse' in fields_set: + csv += str(res['wse']) + csv += ',' + if 'geometry' in fields_set: + csv += res['geometry'].replace('; ', ', ') + csv += ',' + csv += '\n' + data['hits'] = i + data['features'] = csv + return csv + + +def lambda_handler(event, context): + """ + This function queries the database for relevant results + """ + + feature = event['body']['feature'] + subsetpolygon = event['body']['subsetpolygon'] + start_time = event['body']['start_time'] + end_time = event['body']['end_time'] + output = event['body']['output'] + fields = event['body']['fields'] + + results = getlakessubset_get(feature, subsetpolygon, start_time, end_time, output, fields) + + data = {} + + status = "200 OK" + + data['status'] = status + data['time'] = str(10) + " ms." + data['hits'] = 10 + + data['search on'] = { + "parameter": "identifier", + "exact": "exact", + "page_number": 0, + "page_size": 20 + } + + data['results'] = results + + return data diff --git a/hydrocronapi/controllers/lakestimeseries.py b/hydrocronapi/controllers/lakestimeseries.py new file mode 100644 index 0000000..e2601f0 --- /dev/null +++ b/hydrocronapi/controllers/lakestimeseries.py @@ -0,0 +1,215 @@ +# pylint: disable=duplicate-code +# pylint: disable=W0613 +# pylint: disable=R1702 +# pylint: disable=R0912 +"""Module defining Lambda workflow for subset endpoint.""" + +import logging +import time +from datetime import datetime +from typing import Generator + +from hydrocronapi.controllers.db import db + +logger = logging.getLogger() + + +def getlakestimeseries_get(feature, feature_id, start_time, end_time, output, fields) -> object: # noqa: E501 + """Get Timeseries for a particular Reach, Node, or LakeID + + Get Timeseries for a particular Reach, Node, or LakeID # noqa: E501 + + :param feature: Data requested for Reach or Node or Lake + :type feature: str + :param feature_id: ID of the feature to retrieve + :type feature_id: str + :param start_time: Start time of the timeseries + :type start_time: str + :param end_time: End time of the timeseries + :type end_time: str + :param cycleavg: Perform cycle average on the time series + :type cycleavg: bool + :param output: Format of the data returned + :type output: str + + :rtype: None + """ + + start_time = start_time.replace("T", " ")[0:19] + end_time = end_time.replace("T", " ")[0:19] + start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S") + end_time = datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S") + + start = time.time() + results = db.get_series_by_feature_id(feature.lower(), feature_id, start_time, end_time) + + end = time.time() + + data = "" + if output == 'geojson': + data = format_json(results, feature_id, start_time, end_time, round((end - start) * 1000, 3)) + if output == 'csv': + data = format_csv(results, feature_id, start_time, end_time, fields, round((end - start) * 1000, 3)) + + return data + + +def format_json(results: Generator, feature_id, start_time, end_time, elapsed_time): + """ + + Parameters + ---------- + results + feature_id + elapsed_time + + Returns + ------- + + """ + # Fetch all results + if 'Items' in results: + results = results['Items'] + else: + results = [results['Item']] + + data = {} + + if results is None: + data['error'] = f"404: Results with the specified Feature ID {feature_id} were not found." + elif len(results) > 5750000: + data['error'] = f'413: Query exceeds 6MB with {len(results)} hits.' + + else: + data['status'] = "200 OK" + data['time'] = str(elapsed_time) + " ms." + # data['search on'] = {"reach_id": feature_id} + data['type'] = "FeatureCollection" + data['features'] = [] + i = 0 + + for res in results: + if res['time'] != '-999999999999': # and (res['width'] != '-999999999999')): + time_str = datetime.strptime(res['time_str'].replace("T", " ")[0:19], "%Y-%m-%d %H:%M:%S") + if start_time <= time_str <= end_time: + feature = {'properties': {}, 'geometry': {}, 'type': "Feature"} + feature['geometry']['coordinates'] = [] + feature_type = '' + if 'POINT' in res['geometry']: + geometry = res['geometry'].replace('POINT (', '').replace(')', '') + geometry = geometry.replace('"', '') + geometry = geometry.replace("'", "") + feature_type = 'Point' + if 'LINESTRING' in res['geometry']: + geometry = res['geometry'].replace('LINESTRING (', '').replace(')', '') + geometry = geometry.replace('"', '') + geometry = geometry.replace("'", "") + feature_type = 'LineString' + feature['geometry']['type'] = feature_type + for pol in geometry.split(", "): + (var_x, var_y) = pol.split(" ") + if feature_type == 'LineString': + feature['geometry']['coordinates'].append([float(var_x), float(var_y)]) + if feature_type == 'Point': + feature['geometry']['coordinates'] = [float(var_x), float(var_y)] + i += 1 + feature['properties']['time'] = datetime.fromtimestamp(float(res['time']) + 946710000).strftime( + "%Y-%m-%d %H:%M:%S") + feature['properties']['reach_id'] = float(res['reach_id']) + feature['properties']['wse'] = float(res['wse']) + feature['properties']['slope'] = float(res['slope']) + data['features'].append(feature) + data['hits'] = i + print(data) + return data + + +def format_csv(results: Generator, feature_id, start_time, end_time, fields, elapsed_time): + """ + + Parameters + ---------- + results + feature_id + fields + + Returns + ------- + + """ + # Fetch all results + if 'Items' in results: + results = results['Items'] + else: + results = [results['Item']] + + data = {} + + if results is None: + data['error'] = f"404: Results with the specified Feature ID {feature_id} were not found." + elif len(results) > 5750000: + data['error'] = f'413: Query exceeds 6MB with {len(results)} hits.' + + else: + data['status'] = "200 OK" + data['time'] = str(elapsed_time) + " ms." + # data['search on'] = {"reach_id": feature_id} + data['type'] = "FeatureCollection" + data['features'] = [] + i = 0 + csv = fields + '\n' + fields_set = fields.split(", ") + for res in results: + if res['time'] != '-999999999999': # and (res['width'] != '-999999999999')): + time_str = datetime.strptime(res['time_str'].replace("T", " ")[0:19], "%Y-%m-%d %H:%M:%S") + if start_time <= time_str <= end_time: + if 'reach_id' in fields_set: + csv += res['reach_id'] + csv += ',' + if 'time_str' in fields_set: + csv += res['time_str'] + csv += ',' + if 'wse' in fields_set: + csv += str(res['wse']) + csv += ',' + if 'geometry' in fields_set: + csv += res['geometry'].replace('; ', ', ') + csv += ',' + csv += '\n' + data['hits'] = i + data['features'] = csv + return csv + + +def lambda_handler(event, context): + """ + This function queries the database for relevant results + """ + + feature = event['body']['feature'] + feature_id = event['body']['reach_id'] + start_time = event['body']['start_time'] + end_time = event['body']['end_time'] + output = event['body']['output'] + fields = event['body']['fields'] + + results = getlakestimeseries_get(feature, feature_id, start_time, end_time, output, fields) + + data = {} + + status = "200 OK" + + data['status'] = status + data['time'] = str(10) + " ms." + data['hits'] = 10 + + data['search on'] = { + "parameter": "identifier", + "exact": "exact", + "page_number": 0, + "page_size": 20 + } + + data['results'] = results + + return data diff --git a/hydrocronapi/swagger/swagger.yaml b/hydrocronapi/swagger/swagger.yaml index 26a07ac..cb86a91 100644 --- a/hydrocronapi/swagger/swagger.yaml +++ b/hydrocronapi/swagger/swagger.yaml @@ -23,7 +23,7 @@ paths: explode: true schema: type: string - enum: [ "Reach", "Lake", "Node"] + enum: [ "Reach", "Node"] example: Reach - name: feature_id in: query @@ -123,7 +123,7 @@ paths: explode: true schema: type: string - enum: [ "Reach", "Lake", "Node"] + enum: [ "Reach", "Node"] example: Reach - name: subsetpolygon in: query @@ -210,3 +210,204 @@ paths: type: string x-openapi-router-controller: hydrocronapi.controllers.subset + /lakestimeseries: + get: + summary: "Get Timeseries for a particular Reach, Node, or LakeID" + description: "Get Timeseries for a particular Reach, Node, or LakeID" + operationId: getlakestimeseries_get + parameters: + - name: feature + in: query + description: Data requested for Reach or Node or Lake + required: false + style: form + explode: true + schema: + type: string + enum: ["Lake"] + example: Lake + - name: feature_id + in: query + description: ID of the feature to retrieve in format CBBTTTSNNNNNN (i.e. 74297700000000) + required: true + style: form + explode: true + schema: + type: string + example: 71224100223 + - name: start_time + in: query + description: Start time of the timeseries + required: true + style: form + explode: true + schema: + type: string + format: date-time + example: 2022-08-04T00:00:00Z + - name: end_time + in: query + description: End time of the timeseries + required: true + style: form + explode: true + schema: + type: string + format: date-time + example: 2023-08-23T00:00:00Z + - name: output + in: query + description: Format of the data returned + required: false + style: form + explode: true + schema: + type: string + enum: [ "csv", "geojson"] + default: geojson + example: geojson + - name: fields + in: query + description: Format of the data returned + required: false + style: form + explode: true + schema: + type: string + default: reach_id, time_str, wse, geometry + example: reach_id, time_str, wse, geometry + responses: + "200": + description: OK + content: + text/csv: + schema: + type: array + items: + type: string + "400": + description: "400 error. The specified URL is invalid (does not exist)." + content: + text/csv: + schema: + type: array + items: + type: string + "404": + description: "404 error. An entry with the specified region was not found." + content: + text/csv: + schema: + type: array + items: + type: string + "413": + description: "413 error. Your query has returned is too large." + content: + text/csv: + schema: + type: array + items: + type: string + x-openapi-router-controller: hydrocronapi.controllers.lakestimeseries + /lakestimeseriesSubset: + get: + summary: Subset by time series for a given spatial region + description: "Get Timeseries for a particular Reach, Node, or LakeID" + operationId: getlakessubset_get + parameters: + - name: feature + in: query + description: Data requested for Reach or Node or Lake + required: false + style: form + explode: true + schema: + type: string + enum: ["Lake"] + example: Lake + - name: subsetpolygon + in: query + description: GEOJSON of the subset area + required: false + style: form + explode: true + schema: + type: string + example: '{"features": [{"type": "Feature","geometry": {"coordinates": [[-95.6499095054704,50.323685647314554],[-95.3499095054704,50.323685647314554],[-95.3499095054704,50.19088502467528],[-95.6499095054704,50.19088502467528],[-95.6499095054704,50.323685647314554]],"type": "LineString"},"properties": {}}],"type": "FeatureCollection"}' + - name: start_time + in: query + description: Start time of the timeseries + required: true + style: form + explode: true + schema: + type: string + format: date-time + example: 2022-08-04T00:00:00Z + - name: end_time + in: query + description: End time of the timeseries + required: true + style: form + explode: true + schema: + type: string + format: date-time + example: 2023-08-23T00:00:00Z + - name: output + in: query + description: Format of the data returned + required: false + style: form + explode: true + schema: + type: string + enum: [ "csv", "geojson"] + default: geojson + example: geojson + - name: fields + in: query + description: Format of the data returned + required: false + style: form + explode: true + schema: + type: string + default: reach_id, time_str, wse, geometry + example: reach_id, time_str, wse, geometry + responses: + "200": + description: OK + content: + text/csv: + schema: + type: array + items: + type: string + "400": + description: "400 error. The specified URL is invalid (does not exist)." + content: + text/csv: + schema: + type: array + items: + type: string + "404": + description: "404 error. An entry with the specified region was not found." + content: + text/csv: + schema: + type: array + items: + type: string + "413": + description: "413 error. Your query has returned is too large." + content: + text/csv: + schema: + type: array + items: + type: string + x-openapi-router-controller: hydrocronapi.controllers.lakessubset +