diff --git a/data-loading-service/app/config.py b/data-loading-service/app/config.py index 8eba4f1..c216fd7 100644 --- a/data-loading-service/app/config.py +++ b/data-loading-service/app/config.py @@ -46,7 +46,7 @@ def set_db_schema(): class Config: - BASE_URL = "https://api.metro.net" + BASE_URL = os.environ.get('BASE_URL') TARGET_DB_SCHEMA = set_db_schema() API_DB_URI = os.environ.get('API_DB_URI') SECRET_KEY = os.environ.get('HASH_KEY') @@ -66,12 +66,5 @@ class Config: LOGZIO_TOKEN = os.environ.get('LOGZIO_TOKEN') LOGZIO_URL = os.environ.get('LOGZIO_URL') RUNNING_ENV = os.environ.get('RUNNING_ENV') - MAIL_USERNAME = os.environ.get('MAIL_USERNAME') - MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD') - MAIL_FROM = os.environ.get('MAIL_FROM') - MAIL_PORT = os.environ.get('MAIL_PORT') - MAIL_SERVER = os.environ.get('MAIL_SERVER') - MAIL_TLS = "True" - MAIL_SSL = "False" USE_CREDENTIALS = "True" VALIDATE_CERTS = "True" \ No newline at end of file diff --git a/data-loading-service/app/main.py b/data-loading-service/app/main.py index 1712f01..e149036 100644 --- a/data-loading-service/app/main.py +++ b/data-loading-service/app/main.py @@ -12,14 +12,20 @@ lock = threading.Lock() +def retry_on_failure(task, retries=3, delay=15): + for i in range(retries): + try: + task() + break + except Exception as e: + print(f'Error on attempt {i+1}: {str(e)}') + time.sleep(delay) + @crython.job(second='*/15') def gtfs_rt_scheduler(): - with lock: - if lock.locked(): - try: - gtfs_rt_helper.update_gtfs_realtime_data() - except Exception as e: - print('Error updating GTFS-RT data: ' + str(e)) + if not lock.locked(): + with lock: + retry_on_failure(gtfs_rt_helper.update_gtfs_realtime_data) @crython.job(expr='@daily') def go_pass_data_scheduler(): diff --git a/data-loading-service/app/utils/gtfs_static_helper.py b/data-loading-service/app/utils/gtfs_static_helper.py index 0a1ae00..f57e9e3 100644 --- a/data-loading-service/app/utils/gtfs_static_helper.py +++ b/data-loading-service/app/utils/gtfs_static_helper.py @@ -1,17 +1,12 @@ from calendar import calendar import pandas as pd -import json from pathlib import Path from sqlalchemy import create_engine -# from sqlalchemy.orm import Session,sessionmaker from config import Config import geopandas as gpd from .database_connector import * import requests from io import StringIO -# from .utils.log_helper import * -# engine = create_engine(Config.API_DB_URI, echo=False,executemany_mode="values") -# Session = sessionmaker(autocommit=False, autoflush=False, bind=engine) CALENDAR_DATES_URL_BUS = 'https://gitlab.com/LACMTA/gtfs_bus/-/raw/weekly-updated-service/calendar_dates.txt' # CALENDAR_DATES_URL_RAIL = 'https://gitlab.com/LACMTA/gtfs_rail/-/raw/weekly-updated-service/calendar_dates.txt' diff --git a/fastapi/app/config.py b/fastapi/app/config.py index e2d0c1c..569809b 100644 --- a/fastapi/app/config.py +++ b/fastapi/app/config.py @@ -59,15 +59,13 @@ def get_pgbouncer_uri(original_uri): return pgbouncer_uri class Config: - BASE_URL = "https://api.metro.net" + BASE_URL = os.environ.get('BASE_URL') REDIS_URL = os.environ.get('REDIS_URL', 'redis://redis:6379') - TARGET_DB_SCHEMA = "metro_api" + TARGET_DB_SCHEMA = os.environ.get('TARGET_DB_SCHEMA') API_DB_URI = get_pgbouncer_uri(os.environ.get('API_DB_URI')) SECRET_KEY = os.environ.get('HASH_KEY') ALGORITHM = os.environ.get('HASHING_ALGORITHM') ACCESS_TOKEN_EXPIRE_MINUTES = 30 - SWIFTLY_AUTH_KEY_BUS = os.environ.get('SWIFTLY_AUTH_KEY_BUS') - SWIFTLY_AUTH_KEY_RAIL = os.environ.get('SWIFTLY_AUTH_KEY_RAIL') SERVER = os.environ.get('FTP_SERVER') USERNAME = os.environ.get('FTP_USERNAME') PASS = os.environ.get('FTP_PASS')