Merge pull request #418 from LACMTA:2023-api-optimization

Update deployment configurations for AWS Lightsail
LACMTA · Nov 28, 2023 · 7ba8207 · 7ba8207
2 parents c19cb73 + df2577a
commit 7ba8207
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 7 deletions.
diff --git a/.github/workflows/dev-aws.yml b/.github/workflows/dev-aws.yml
@@ -96,7 +96,7 @@ jobs:
           MAX_RETRIES=5
           RETRY_DELAY=5
           count=0
-          until aws lightsail create-container-service-deployment --service-name dev-metro-api-v2  --containers '{"redis": {"image": "redis:latest", "environment": {}, "ports": {"6379": "TCP"}}, "fastapi": {"image":"lacmta/metro-api-v2:fastapi", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "https://listener.logz.io:8071", "RUNNING_ENV": "dev", "REDIS_URL": "redis://dev-metro-api-v2.service.local:6379"}, "ports": {"80": "HTTP"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}'
+          until aws lightsail create-container-service-deployment --service-name dev-metro-api-v2  --containers '{"redis": {"image": "redis:latest", "environment": {}, "ports": {"6379": "TCP"}}, "fastapi": {"image":"lacmta/metro-api-v2:fastapi", "environment": { "BASE_URL": "https://dev-metro-api-v2.ofhq3vd1r7une.us-west-2.cs.amazonlightsail.com","FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "https://listener.logz.io:8071", "BASE_URL": "https://dev-metro-api-v2.ofhq3vd1r7une.us-west-2.cs.amazonlightsail.com","RUNNING_ENV": "dev", "REDIS_URL": "redis://dev-metro-api-v2.service.local:6379"}, "ports": {"80": "HTTP"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}'
           do
             if [[ "$count" -lt "$MAX_RETRIES" ]]; then
               count=$((count+1))

diff --git a/.github/workflows/prod-aws.yml b/.github/workflows/prod-aws.yml
@@ -99,7 +99,7 @@ jobs:
           MAX_RETRIES=5
           RETRY_DELAY=5
           count=0
-          until aws lightsail create-container-service-deployment --service-name metro-api-v2 --containers '{"redis": {"image": "redis:latest", "environment": {}, "ports": {"6379": "TCP"}}, "fastapi": {"image":"lacmta/metro-api-v2:fastapi", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "https://listener.logz.io:8071", "RUNNING_ENV": "prod", "REDIS_URL": "redis://metro-api-v2.service.local:6379"}, "ports": {"80": "HTTP"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}'
+          until aws lightsail create-container-service-deployment --service-name metro-api-v2 --containers '{"redis": {"image": "redis:latest", "environment": {}, "ports": {"6379": "TCP"}}, "fastapi": {"image":"lacmta/metro-api-v2:fastapi", "environment": {"BASE_URL": "https://api.metro.net","FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "https://listener.logz.io:8071","BASE_URL": "https://api.metro.net", "RUNNING_ENV": "prod", "REDIS_URL": "redis://metro-api-v2.service.local:6379"}, "ports": {"80": "HTTP"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}'
           do
             if [[ "$count" -lt "$MAX_RETRIES" ]]; then
               count=$((count+1))

diff --git a/data-loading-service/app/main.py b/data-loading-service/app/main.py
@@ -9,13 +9,17 @@
 import pandas as pd
 
 import crython
-# import schedule
+
+lock = threading.Lock()
+
 @crython.job(second='*/15')
 def gtfs_rt_scheduler():
-    try:
-        gtfs_rt_helper.update_gtfs_realtime_data()
-    except Exception as e:
-        print('Error updating GTFS-RT data: ' + str(e))
+    with lock:
+        if lock.locked():
+            try:
+                gtfs_rt_helper.update_gtfs_realtime_data()
+            except Exception as e:
+                print('Error updating GTFS-RT data: ' + str(e))
 
 @crython.job(expr='@daily')
 def go_pass_data_scheduler():