From 89c028b533707db94b12119080d1053800d295cb Mon Sep 17 00:00:00 2001 From: Andre LeBlanc Date: Mon, 30 Dec 2024 20:28:03 +0000 Subject: [PATCH] Add logic from gitlab version of plugins --- sarracenia/flowcb/poll/airnow.py | 66 +++++++++++++++++--------------- sarracenia/flowcb/poll/odata.py | 17 ++++++++ 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/sarracenia/flowcb/poll/airnow.py b/sarracenia/flowcb/poll/airnow.py index 4e13edb8e..39008c0a3 100755 --- a/sarracenia/flowcb/poll/airnow.py +++ b/sarracenia/flowcb/poll/airnow.py @@ -30,35 +30,39 @@ def poll(self): sleep = self.o.scheduled_interval gathered_messages = [] - for Hours in range(1, 3): - last_hour_date_time = datetime.datetime.now() - datetime.timedelta( - hours=Hours) - Filename = 'HourlyData_%s.dat' % last_hour_date_time.strftime( - '%Y%m%d%H') - logger.debug("poll_airnow_http Filename: %s" % Filename) - URL = self.o.pollUrl + '/' + Filename - logger.info('INFO %s ' % URL) - #resp = requests.get(self.o.pollUrl + '/' + Filename) - resp = requests.get(URL) - if resp.ok: - mtime = datetime.datetime.strptime(resp.headers['last-modified'],\ - '%a, %d %b %Y %H:%M:%S %Z') - last_poll = datetime.datetime.utcnow() + datetime.timedelta( - seconds=-sleep) - logger.info(mtime) - logger.info(last_poll) - - fakeStat = paramiko.SFTPAttributes() - fakeStat.st_size = int(resp.headers['content-length']) - - # convert datetime to numeric timestamp from beginning of POSIX epoch. - fakeStat.st_mtime = mtime.timestamp() - fakeStat.st_atime = mtime.timestamp() - fakeStat.st_mode = 0o644 - - m = sarracenia.Message.fromFileInfo(Filename, self.o, fakeStat) - gathered_messages.append(m) - - logger.info('mtime: %s last_pollL %s' % (mtime, last_poll)) - + try: + for Hours in range(1, 3): + last_hour_date_time = datetime.datetime.now() - datetime.timedelta( + hours=Hours) + Filename = 'HourlyData_%s.dat' % last_hour_date_time.strftime( + '%Y%m%d%H') + logger.debug("poll_airnow_http Filename: %s" % Filename) + URL = self.o.pollUrl + '/' + Filename + logger.info('INFO %s ' % URL) + #resp = requests.get(self.o.pollUrl + '/' + Filename) + resp = requests.get(URL) + if resp.ok: + mtime = datetime.datetime.strptime(resp.headers['last-modified'],\ + '%a, %d %b %Y %H:%M:%S %Z') + last_poll = datetime.datetime.utcnow() + datetime.timedelta( + seconds=-sleep) + logger.info(mtime) + logger.info(last_poll) + + fakeStat = paramiko.SFTPAttributes() + fakeStat.st_size = int(resp.headers['content-length']) + + # convert datetime to numeric timestamp from beginning of POSIX epoch. + fakeStat.st_mtime = mtime.timestamp() + fakeStat.st_atime = mtime.timestamp() + fakeStat.st_mode = 0o644 + + m = sarracenia.Message.fromFileInfo(Filename, self.o, fakeStat) + gathered_messages.append(m) + + logger.info('mtime: %s last_pollL %s' % (mtime, last_poll)) + except Exception as e: + logger.error(f"Poll failed: {e}") + logger.debug("Exception details:", exc_info=True) + return gathered_messages diff --git a/sarracenia/flowcb/poll/odata.py b/sarracenia/flowcb/poll/odata.py index 09a370225..698332e03 100644 --- a/sarracenia/flowcb/poll/odata.py +++ b/sarracenia/flowcb/poll/odata.py @@ -226,6 +226,23 @@ def parse_json_to_msg(self, jdata): """ msg = None + # {'@odata.mediaContentType': 'application/octet-stream', + # 'Id': 'd86d00a2-58bc-4603-9e6a-28bc571d79a6', + # 'Name': 'S1A_IW_GRDH_1SDV_20230425T123926_20230425T123951_048253_05CD62_EA4E_COG.SAFE', + # 'ContentType': 'application/octet-stream', + # 'ContentLength': 1229909379, + # 'OriginDate': '2023-05-06T12:36:57.469Z', + # 'PublicationDate': '2023-05-06T12:42:38.119Z', + # 'ModificationDate': '2023-05-06T12:42:38.119Z', + # 'Online': True, + # 'EvictionDate': '', + # 'S3Path': '/eodata/Sentinel-1/SAR/IW_GRDH_1S-COG/2023/04/25/S1A_IW_GRDH_1SDV_20230425T123926_20230425T123951_048253_05CD62_EA4E_COG.SAFE', + # 'Checksum': [{}], + # 'ContentDate': {'Start': '2023-04-25T12:39:26.902Z', 'End': '2023-04-25T12:39:51.900Z'}, + # 'Footprint': "geography'SRID=4326;POLYGON ((79.757111 29.038794, 82.362259 29.452579, 82.123009 30.965038, 79.47876 30.555256, 79.757111 29.038794))'", + # 'GeoFootprint': {'type': 'Polygon', 'coordinates': [[[79.757111, 29.038794], [82.362259, 29.452579], [82.123009, 30.965038], [79.47876, 30.555256], [79.757111, 29.038794]]]} + # } + data_path = self.o.post_urlTemplate.replace('--PRODUCT_ID--', jdata['Id']) msg = sarracenia.Message.fromFileInfo(data_path, self.o)