API_DB_URI refactoring

.env_sample

@@ -6,7 +6,7 @@ SWIFTLY_AUTH_KEY_RAIL=
 AWS_ACCESS_KEY_ID=
 ACCESS_SECRET_KEY=
 SWIFTLY_AUTH_KEY=
-URI=
+API_DB_URI=
 HASH_KEY=
 HASHING_ALGORITHM=
 LOGZIO_TOKEN=

.github/workflows/dev-aws.yml

@@ -81,7 +81,7 @@ jobs:
         run: >
           aws lightsail create-container-service-deployment 
           --service-name dev-metro-api-v2 
-          --containers '{"fastapi":{"image":"lacmta/metro-api-v2:fastapi","ports":{"80":"HTTP"},"environment":{"FTP_PASS":"${{secrets.FTP_PASS }}","FTP_SERVER":"${{secrets.FTP_SERVER }}","FTP_USERNAME":"${{secrets.FTP_USERNAME }}","SWIFTLY_AUTH_KEY_BUS":"${{secrets.SWIFTLY_AUTH_KEY_BUS}}","SWIFTLY_AUTH_KEY_RAIL":"${{secrets.SWIFTLY_AUTH_KEY_RAIL}}","URI":"${{secrets.API_DB_URI}}","HASH_KEY":"${{secrets.HASH_KEY}}","HASHING_ALGORITHM":"${{secrets.HASHING_ALGORITHM}}","LOGZIO_TOKEN":"${{secrets.LOGZIO_TOKEN}}","LOGZIO_URL":"https://listener.logz.io:8071","RUNNING_ENV":"dev","MAIL_USERNAME":"${{secrets.MAIL_USERNAME}}","MAIL_PASSWORD":"${{secrets.MAIL_PASSWORD}}","MAIL_SERVER":"${{secrets.MAIL_SERVER}}"}},"data-loading-service":{"image":"lacmta/metro-api-v2:data-loading-service","environment":{"FTP_PASS":"${{secrets.FTP_PASS }}","FTP_SERVER":"${{secrets.FTP_SERVER }}","FTP_USERNAME":"${{secrets.FTP_USERNAME }}","SWIFTLY_AUTH_KEY_BUS":"${{secrets.SWIFTLY_AUTH_KEY_BUS}}","SWIFTLY_AUTH_KEY_RAIL":"${{secrets.SWIFTLY_AUTH_KEY_RAIL}}","URI":"${{secrets.URI}}","HASH_KEY":"${{secrets.HASH_KEY}}","HASHING_ALGORITHM":"${{secrets.HASHING_ALGORITHM}}","LOGZIO_TOKEN":"${{secrets.LOGZIO_TOKEN}}","LOGZIO_URL":"https://listener.logz.io:8071","RUNNING_ENV":"dev","MAIL_USERNAME":"${{secrets.MAIL_USERNAME}}","MAIL_PASSWORD":"${{secrets.MAIL_PASSWORD}}","MAIL_SERVER":"${{secrets.MAIL_SERVER}}"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}'
+          --containers '{"fastapi":{"image":"lacmta/metro-api-v2:fastapi","ports":{"80":"HTTP"},"environment":{"FTP_PASS":"${{secrets.FTP_PASS }}","FTP_SERVER":"${{secrets.FTP_SERVER }}","FTP_USERNAME":"${{secrets.FTP_USERNAME }}","SWIFTLY_AUTH_KEY_BUS":"${{secrets.SWIFTLY_AUTH_KEY_BUS}}","SWIFTLY_AUTH_KEY_RAIL":"${{secrets.SWIFTLY_AUTH_KEY_RAIL}}","API_DB_URI":"${{secrets.API_DB_URI}}","HASH_KEY":"${{secrets.HASH_KEY}}","HASHING_ALGORITHM":"${{secrets.HASHING_ALGORITHM}}","LOGZIO_TOKEN":"${{secrets.LOGZIO_TOKEN}}","LOGZIO_URL":"https://listener.logz.io:8071","RUNNING_ENV":"dev","MAIL_USERNAME":"${{secrets.MAIL_USERNAME}}","MAIL_PASSWORD":"${{secrets.MAIL_PASSWORD}}","MAIL_SERVER":"${{secrets.MAIL_SERVER}}"}},"data-loading-service":{"image":"lacmta/metro-api-v2:data-loading-service","environment":{"FTP_PASS":"${{secrets.FTP_PASS }}","FTP_SERVER":"${{secrets.FTP_SERVER }}","FTP_USERNAME":"${{secrets.FTP_USERNAME }}","SWIFTLY_AUTH_KEY_BUS":"${{secrets.SWIFTLY_AUTH_KEY_BUS}}","SWIFTLY_AUTH_KEY_RAIL":"${{secrets.SWIFTLY_AUTH_KEY_RAIL}}","API_DB_URI":"${{secrets.API_DB_URI}}","HASH_KEY":"${{secrets.HASH_KEY}}","HASHING_ALGORITHM":"${{secrets.HASHING_ALGORITHM}}","LOGZIO_TOKEN":"${{secrets.LOGZIO_TOKEN}}","LOGZIO_URL":"https://listener.logz.io:8071","RUNNING_ENV":"dev","MAIL_USERNAME":"${{secrets.MAIL_USERNAME}}","MAIL_PASSWORD":"${{secrets.MAIL_PASSWORD}}","MAIL_SERVER":"${{secrets.MAIL_SERVER}}"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}'
   deploy-documentation:
     runs-on: ubuntu-latest
     name: Deploy Documentation to GitHub Pages

.github/workflows/gtfs-static-updater.yml

@@ -23,5 +23,5 @@ jobs:
       - name: Update Database with latest data🛢
         run: python ./data-loading-service/app/gtfs_static_updater
         env:
-          DB_URI: ${{secrets.URI}}
+          API_DB_URI: ${{secrets.API_DB_URI}}
           TARGET_DB_SCHEMA: "metro_api"

.github/workflows/prod-aws.yml

@@ -81,7 +81,7 @@ jobs:
         run: >
           aws lightsail create-container-service-deployment 
           --service-name metro-api-v2 
-          --containers '{"fastapi":{"image":"lacmta/metro-api-v2:fastapi","ports":{"80":"HTTP"},"environment":{"FTP_PASS":"${{secrets.FTP_PASS }}","FTP_SERVER":"${{secrets.FTP_SERVER }}","FTP_USERNAME":"${{secrets.FTP_USERNAME }}","SWIFTLY_AUTH_KEY_BUS":"${{secrets.SWIFTLY_AUTH_KEY_BUS}}","SWIFTLY_AUTH_KEY_RAIL":"${{secrets.SWIFTLY_AUTH_KEY_RAIL}}","URI":"${{secrets.API_DB_URI}}","HASH_KEY":"${{secrets.HASH_KEY}}","HASHING_ALGORITHM":"${{secrets.HASHING_ALGORITHM}}","LOGZIO_TOKEN":"${{secrets.LOGZIO_TOKEN}}","LOGZIO_URL":"https://listener.logz.io:8071","RUNNING_ENV":"prod","MAIL_USERNAME":"${{secrets.MAIL_USERNAME}}","MAIL_PASSWORD":"${{secrets.MAIL_PASSWORD}}","MAIL_SERVER":"${{secrets.MAIL_SERVER}}"}},"data-loading-service":{"image":"lacmta/metro-api-v2:data-loading-service","environment":{"FTP_PASS":"${{secrets.FTP_PASS }}","FTP_SERVER":"${{secrets.FTP_SERVER }}","FTP_USERNAME":"${{secrets.FTP_USERNAME }}","SWIFTLY_AUTH_KEY_BUS":"${{secrets.SWIFTLY_AUTH_KEY_BUS}}","SWIFTLY_AUTH_KEY_RAIL":"${{secrets.SWIFTLY_AUTH_KEY_RAIL}}","URI":"${{secrets.URI}}","HASH_KEY":"${{secrets.HASH_KEY}}","HASHING_ALGORITHM":"${{secrets.HASHING_ALGORITHM}}","LOGZIO_TOKEN":"${{secrets.LOGZIO_TOKEN}}","LOGZIO_URL":"https://listener.logz.io:8071","RUNNING_ENV":"prod","MAIL_USERNAME":"${{secrets.MAIL_USERNAME}}","MAIL_PASSWORD":"${{secrets.MAIL_PASSWORD}}","MAIL_SERVER":"${{secrets.MAIL_SERVER}}"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}'
+          --containers '{"fastapi":{"image":"lacmta/metro-api-v2:fastapi","ports":{"80":"HTTP"},"environment":{"FTP_PASS":"${{secrets.FTP_PASS }}","FTP_SERVER":"${{secrets.FTP_SERVER }}","FTP_USERNAME":"${{secrets.FTP_USERNAME }}","SWIFTLY_AUTH_KEY_BUS":"${{secrets.SWIFTLY_AUTH_KEY_BUS}}","SWIFTLY_AUTH_KEY_RAIL":"${{secrets.SWIFTLY_AUTH_KEY_RAIL}}","API_DB_URI":"${{secrets.API_DB_URI}}","HASH_KEY":"${{secrets.HASH_KEY}}","HASHING_ALGORITHM":"${{secrets.HASHING_ALGORITHM}}","LOGZIO_TOKEN":"${{secrets.LOGZIO_TOKEN}}","LOGZIO_URL":"https://listener.logz.io:8071","RUNNING_ENV":"prod","MAIL_USERNAME":"${{secrets.MAIL_USERNAME}}","MAIL_PASSWORD":"${{secrets.MAIL_PASSWORD}}","MAIL_SERVER":"${{secrets.MAIL_SERVER}}"}},"data-loading-service":{"image":"lacmta/metro-api-v2:data-loading-service","environment":{"FTP_PASS":"${{secrets.FTP_PASS }}","FTP_SERVER":"${{secrets.FTP_SERVER }}","FTP_USERNAME":"${{secrets.FTP_USERNAME }}","SWIFTLY_AUTH_KEY_BUS":"${{secrets.SWIFTLY_AUTH_KEY_BUS}}","SWIFTLY_AUTH_KEY_RAIL":"${{secrets.SWIFTLY_AUTH_KEY_RAIL}}","API_DB_URI":"${{secrets.API_DB_URI}}","HASH_KEY":"${{secrets.HASH_KEY}}","HASHING_ALGORITHM":"${{secrets.HASHING_ALGORITHM}}","LOGZIO_TOKEN":"${{secrets.LOGZIO_TOKEN}}","LOGZIO_URL":"https://listener.logz.io:8071","RUNNING_ENV":"prod","MAIL_USERNAME":"${{secrets.MAIL_USERNAME}}","MAIL_PASSWORD":"${{secrets.MAIL_PASSWORD}}","MAIL_SERVER":"${{secrets.MAIL_SERVER}}"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}'
 
   deploy-documentation:
     runs-on: ubuntu-latest

data-loading-service/app/config.py

@@ -23,7 +23,7 @@ def set_db_schema():
 class Config:
     BASE_URL = "https://api.metro.net"
     TARGET_DB_SCHEMA = set_db_schema()
-    DB_URI = os.environ.get('URI')
+    API_DB_URI = os.environ.get('API_DB_URI')
     SECRET_KEY = os.environ.get('HASH_KEY')
     ALGORITHM = os.environ.get('HASHING_ALGORITHM')
     ACCESS_TOKEN_EXPIRE_MINUTES  = 30

data-loading-service/app/utils/database_connector.py

@@ -7,7 +7,7 @@
 from config import Config
 # from .utils.log_helper import *
 
-engine = create_engine(Config.DB_URI, echo=False)
+engine = create_engine(Config.API_DB_URI, echo=False)
 
 Session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 

data-loading-service/app/utils/gtfs_rt_helper.py

@@ -41,7 +41,7 @@
 SWIFTLY_GTFS_RT_TRIP_UPDATES = 'gtfs-rt-trip-updates'
 SWIFTLY_GTFS_RT_VEHICLE_POSITIONS = 'gtfs-rt-vehicle-positions'
 
-# engine = create_engine(Config.DB_URI, echo=False,executemany_mode="values")
+# engine = create_engine(Config.API_DB_URI, echo=False,executemany_mode="values")
 
 # Session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 

data-loading-service/app/utils/gtfs_static_helper.py

@@ -10,7 +10,7 @@
 import requests
 from io import StringIO
 # from .utils.log_helper import *
-# engine = create_engine(Config.DB_URI, echo=False,executemany_mode="values")
+# engine = create_engine(Config.API_DB_URI, echo=False,executemany_mode="values")
 # Session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 CALENDAR_DATES_URL_BUS = 'https://gitlab.com/LACMTA/gtfs_bus/-/raw/weekly-updated-service/calendar_dates.txt'

documentation/blog/2023-03-29.mdx

@@ -47,7 +47,7 @@ from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
 
 We then re-did our SqlAlchemy engine to connect using the async url: 
 ``` python
-async_engine = create_async_engine(create_async_uri(Config.DB_URI), echo=False)
+async_engine = create_async_engine(create_async_uri(Config.API_DB_URI), echo=False)
 async_session = sessionmaker(async_engine, expire_on_commit=False, class_=AsyncSession)
 ```
 

fastapi/app/config.py

@@ -13,7 +13,7 @@ class Config:
     BASE_URL = "https://api.metro.net"
     REDIS_URL = "redis://redis:6379"
     TARGET_DB_SCHEMA = "metro_api"
-    DB_URI = os.environ.get('URI')
+    API_DB_URI = os.environ.get('API_DB_URI')
     SECRET_KEY = os.environ.get('HASH_KEY')
     ALGORITHM = os.environ.get('HASHING_ALGORITHM')
     ACCESS_TOKEN_EXPIRE_MINUTES  = 30

fastapi/app/database.py

@@ -12,8 +12,8 @@ def create_async_uri(uri):
     return uri.replace('postgresql', 'postgresql+asyncpg')
 
 
-engine = create_engine(Config.DB_URI, echo=False)
-async_engine = create_async_engine(create_async_uri(Config.DB_URI), echo=False)
+engine = create_engine(Config.API_DB_URI, echo=False)
+async_engine = create_async_engine(create_async_uri(Config.API_DB_URI), echo=False)
 async_session = sessionmaker(async_engine, expire_on_commit=False, class_=AsyncSession)
 Session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 

notebooks/custom-schedule-page.ipynb

@@ -271,7 +271,7 @@
     "# from .utils.log_helper import *\n",
     "\n",
     "from secrets import *\n",
-    "engine = create_engine(Config.URI, echo=False)\n",
+    "engine = create_engine(Config.API_DB_URI, echo=False)\n",
     "\n",
     "Session = sessionmaker(autocommit=False, autoflush=False, bind=engine)\n",
     "\n",

notebooks/data_sandbox.ipynb

@@ -199,7 +199,7 @@
     "# from .utils.log_helper import *\n",
     "\n",
     "from secrets import *\n",
-    "engine = create_engine(Config.URI, echo=False)\n",
+    "engine = create_engine(Config.API_DB_URI, echo=False)\n",
     "\n",
     "Session = sessionmaker(autocommit=False, autoflush=False, bind=engine)\n",
     "\n",

notebooks/manual_gtfs_static_update.ipynb

@@ -48,7 +48,7 @@
     "# from .utils.log_helper import *\n",
     "\n",
     "from secrets import *\n",
-    "engine = create_engine(Config.URI, echo=False)\n",
+    "engine = create_engine(Config.API_DB_URI, echo=False)\n",
     "\n",
     "Session = sessionmaker(autocommit=False, autoflush=False, bind=engine)\n",
     "\n",