Merge pull request #389 from LACMTA:2023-api-optimization

Add Redis initialization and cache configuration
LACMTA · Nov 17, 2023 · b37e12b · b37e12b
2 parents e955745 + 4db35f3
commit b37e12b
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 12 deletions.
diff --git a/.github/workflows/dev-aws.yml b/.github/workflows/dev-aws.yml
@@ -75,8 +75,21 @@ jobs:
             BUILDKIT_CONTEXT_KEEP_GIT_DIR=true
       - name: Push Updated data loading service Docker Image on Lightsail Dev
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: >
-          aws lightsail create-container-service-deployment --service-name data-loading-service --containers '{"data-loading-service": {"image":"lacmta/metro-api-v2:data-loading-service", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "${{secrets.LOGZIO_URL}}", "RUNNING_ENV": "prod"}}}'
+        run: |
+          MAX_RETRIES=5
+          RETRY_DELAY=5
+          count=0
+          until aws lightsail create-container-service-deployment --service-name data-loading-service --containers '{"data-loading-service": {"image":"lacmta/metro-api-v2:data-loading-service", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "${{secrets.LOGZIO_URL}}", "RUNNING_ENV": "prod"}}}'
+          do
+            if [[ "$count" -lt "$MAX_RETRIES" ]]; then
+              count=$((count+1))
+              echo "Attempt $count of $MAX_RETRIES. Retrying after $RETRY_DELAY seconds..."
+              sleep $RETRY_DELAY
+            else
+              echo "All attempts failed. Exiting..."
+              exit 1
+            fi
+          done
       - name: Push Updated FastAPI to Image on Lightsail Dev
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |

diff --git a/.github/workflows/prod-aws.yml b/.github/workflows/prod-aws.yml
@@ -76,10 +76,23 @@ jobs:
           context: ./fastapi/
           build-args: | 
             BUILDKIT_CONTEXT_KEEP_GIT_DIR=true
-      - name: Push Updated data loading service Docker Image on Lightsail Prod
+      - name: Push Updated data loading service Docker Image on Lightsail Dev
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: >
-          aws lightsail create-container-service-deployment --service-name data-loading-service --containers '{"data-loading-service": {"image":"lacmta/metro-api-v2:data-loading-service", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "${{secrets.LOGZIO_URL}}", "RUNNING_ENV": "prod"}}}'
+        run: |
+          MAX_RETRIES=5
+          RETRY_DELAY=5
+          count=0
+          until aws lightsail create-container-service-deployment --service-name data-loading-service --containers '{"data-loading-service": {"image":"lacmta/metro-api-v2:data-loading-service", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "${{secrets.LOGZIO_URL}}", "RUNNING_ENV": "prod"}}}'
+          do
+            if [[ "$count" -lt "$MAX_RETRIES" ]]; then
+              count=$((count+1))
+              echo "Attempt $count of $MAX_RETRIES. Retrying after $RETRY_DELAY seconds..."
+              sleep $RETRY_DELAY
+            else
+              echo "All attempts failed. Exiting..."
+              exit 1
+            fi
+          done
       - name: Push Updated Docker Image on Lightsail Prod
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |

diff --git a/fastapi/app/main.py b/fastapi/app/main.py
@@ -48,6 +48,8 @@
 
 # from redis import asyncio as aioredis
 from enum import Enum
+from aioredis import create_redis_pool
+
 
 # for OAuth2
 from fastapi.security import OAuth2PasswordBearer,OAuth2PasswordRequestForm
@@ -80,17 +82,17 @@
 # Define the redis variable at the top level
 redis = None
 
-def initialize_redis():
+async def initialize_redis():
     global redis
     logging.info(f"Connecting to Redis at {Config.REDIS_URL}")
     for i in range(5):  # Retry up to 5 times
         try:
-            redis = aioredis.from_url(Config.REDIS_URL, socket_connect_timeout=5)
+            redis = await aioredis.create_redis_pool(Config.REDIS_URL)
             break  # If the connection is successful, break out of the loop
         except aioredis.exceptions.ConnectionError as e:
             logging.error(f"Failed to connect to Redis: {e}")
             if i < 4:  # If this was not the last attempt, wait a bit before retrying
-                time.sleep(5)  # Wait for 5 seconds
+                await asyncio.sleep(5)  # Wait for 5 seconds
             else:  # If this was the last attempt, re-raise the exception
                 raise
 
@@ -737,11 +739,9 @@ async def get_all_routes():
 
 @app.on_event("startup")
 async def startup_event():
-    # initialize_redis()
-    # app.state.redis = await aioredis.from_url(, socket_connect_timeout=5)
-    redis = RedisBackend(Config.REDIS_URL)
+    redis_pool = await aioredis.create_redis_pool(Config.REDIS_URL)
+    redis = RedisBackend(redis_pool)
     FastAPICache.init(backend=redis, prefix="fastapi-cache")
-    # await app.state.redis.flushdb()  # Add this line to flush the Redis database
     uvicorn_access_logger = logging.getLogger("uvicorn.access")
     uvicorn_error_logger = logging.getLogger("uvicorn.error")
     logger = logging.getLogger("uvicorn.app")