diff --git a/.github/workflows/dev-aws.yml b/.github/workflows/dev-aws.yml index 5780e33..a4126b6 100644 --- a/.github/workflows/dev-aws.yml +++ b/.github/workflows/dev-aws.yml @@ -81,7 +81,7 @@ jobs: run: > aws lightsail create-container-service-deployment --service-name dev-metro-api-v2 - --containers '{"redis": {"image": "redis:latest", "environment": {}, "ports": {"6379": "HTTP"}}, "prometheus": {"image": "prom/prometheus:latest", "environment": {}, "ports": {"9090": "HTTP"}}, "grafana": {"image": "grafana/grafana:latest", "environment": {}, "ports": {"3000": "HTTP"}}, "data-loading-service": {"image":"lacmta/metro-api-v2:data-loading-service", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "${{secrets.LOGZIO_URL}}", "RUNNING_ENV": "dev"}}, "fastapi": {"image":"lacmta/metro-api-v2:fastapi", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "https://listener.logz.io:8071", "RUNNING_ENV": "dev"}, "ports": {"80": "HTTP"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}' + --containers '{"redis": {"image": "redis:latest", "environment": {}, "ports": {"6379": "HTTP"}}, "prometheus": {"image": "prom/prometheus:latest", "environment": {}, "ports": {"9090": "HTTP"}}, "grafana": {"image": "grafana/grafana:latest", "environment": {}, "ports": {"3000": "HTTP"}}, "data-loading-service": {"image":"lacmta/metro-api-v2:data-loading-service", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "${{secrets.LOGZIO_URL}}", "RUNNING_ENV": "dev"}}, "fastapi": {"image":"lacmta/metro-api-v2:fastapi", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "https://listener.logz.io:8071", "RUNNING_ENV": "dev", "REDIS_URL": "redis://redis.local:6379"}, "ports": {"80": "HTTP"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}' deploy-documentation: runs-on: ubuntu-latest name: Deploy Documentation to GitHub Pages diff --git a/.github/workflows/prod-aws.yml b/.github/workflows/prod-aws.yml index ccb25fd..c0413bf 100644 --- a/.github/workflows/prod-aws.yml +++ b/.github/workflows/prod-aws.yml @@ -81,8 +81,7 @@ jobs: run: > aws lightsail create-container-service-deployment --service-name metro-api-v2 - --containers '{"redis": {"image": "redis:latest", "environment": {}, "ports": {"6379": "HTTP"}}, "prometheus": {"image": "prom/prometheus:latest", "environment": {}, "ports": {"9090": "HTTP"}}, "grafana": {"image": "grafana/grafana:latest", "environment": {}, "ports": {"3000": "HTTP"}}, "data-loading-service": {"image":"lacmta/metro-api-v2:data-loading-service", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "${{secrets.LOGZIO_URL}}", "RUNNING_ENV": "prod"}}, "fastapi": {"image":"lacmta/metro-api-v2:fastapi", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "https://listener.logz.io:8071", "RUNNING_ENV": "prod"}, "ports": {"80": "HTTP"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}' - deploy-documentation: + --containers '{"redis": {"image": "redis:latest", "environment": {}, "ports": {"6379": "HTTP"}}, "prometheus": {"image": "prom/prometheus:latest", "environment": {}, "ports": {"9090": "HTTP"}}, "grafana": {"image": "grafana/grafana:latest", "environment": {}, "ports": {"3000": "HTTP"}}, "data-loading-service": {"image":"lacmta/metro-api-v2:data-loading-service", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "${{secrets.LOGZIO_URL}}", "RUNNING_ENV": "prod"}}, "fastapi": {"image":"lacmta/metro-api-v2:fastapi", "environment": {"FTP_SERVER": "${{secrets.FTP_SERVER}}", "FTP_USERNAME": "${{secrets.FTP_USERNAME}}", "FTP_PASS": "${{secrets.FTP_PASS}}", "SWIFTLY_AUTH_KEY_BUS": "${{secrets.SWIFTLY_AUTH_KEY_BUS}}", "SWIFTLY_AUTH_KEY_RAIL": "${{secrets.SWIFTLY_AUTH_KEY_RAIL}}", "AWS_ACCESS_KEY_ID": "${{secrets.AWS_ACCESS_KEY_ID}}", "ACCESS_SECRET_KEY": "${{secrets.ACCESS_SECRET_KEY}}", "SWIFTLY_AUTH_KEY": "${{secrets.SWIFTLY_AUTH_KEY}}", "API_DB_URI": "${{secrets.API_DB_URI}}", "HASH_KEY": "${{secrets.HASH_KEY}}", "HASHING_ALGORITHM": "${{secrets.HASHING_ALGORITHM}}", "LOGZIO_TOKEN": "${{secrets.LOGZIO_TOKEN}}", "LOGZIO_URL": "https://listener.logz.io:8071", "RUNNING_ENV": "prod", "REDIS_URL": "redis://redis.local:6379"}, "ports": {"80": "HTTP"}}}' --public-endpoint '{"containerName": "fastapi","containerPort": 80,"healthCheck":{ "healthyThreshold": 2,"unhealthyThreshold": 2,"timeoutSeconds": 2,"intervalSeconds": 5,"path": "/","successCodes": "200-499"}}' deploy-documentation: runs-on: ubuntu-latest name: Deploy Documentation to GitHub Pages steps: diff --git a/fastapi/app/config.py b/fastapi/app/config.py index f209ed5..c4869d3 100644 --- a/fastapi/app/config.py +++ b/fastapi/app/config.py @@ -44,7 +44,7 @@ def get_version_tag_from_online_github_repo(): class Config: BASE_URL = "https://api.metro.net" - REDIS_URL = "redis://redis:6379" + REDIS_URL = os.environ.get('REDIS_URL', 'redis://redis:6379') TARGET_DB_SCHEMA = "metro_api" API_DB_URI = os.environ.get('API_DB_URI') SECRET_KEY = os.environ.get('HASH_KEY') diff --git a/fastapi/app/crud.py b/fastapi/app/crud.py index 9ae2811..7015f69 100644 --- a/fastapi/app/crud.py +++ b/fastapi/app/crud.py @@ -50,7 +50,7 @@ from sqlalchemy.orm import Session from sqlalchemy.orm.decl_api import DeclarativeMeta -redis = aioredis.from_url("redis://redis:6379", socket_connect_timeout=5) +redis = aioredis.from_url(Config.REDIS_URL, socket_connect_timeout=5) # import sqlalchemy def asdict(obj): diff --git a/fastapi/app/main.py b/fastapi/app/main.py index 3bc4669..0e6f3f1 100644 --- a/fastapi/app/main.py +++ b/fastapi/app/main.py @@ -30,7 +30,6 @@ from sqlalchemy import false, distinct, inspect from sqlalchemy.orm import aliased - from sqlalchemy.future import select from sqlalchemy.exc import SQLAlchemyError @@ -81,7 +80,7 @@ ) # Create a Redis connection pool -redis = aioredis.from_url("redis://localhost:6379", decode_responses=True, encoding='utf-8', socket_connect_timeout=5) +redis = aioredis.from_url(Config.REDIS_URL, socket_connect_timeout=5) async def get_data(db: Session, key: str, fetch_func): # Get data from Redis @@ -192,15 +191,15 @@ def filter(self, record): inspector = inspect(engine) - try: for table_name in models.Base.metadata.tables.keys(): - if not engine.dialect.has_table(engine, table_name): + if not inspector.has_table(table_name): models.Base.metadata.tables[table_name].create(bind=engine) except SQLAlchemyError as e: print(f"An error occurred while creating the tables: {e}") + app = FastAPI(openapi_tags=tags_metadata,docs_url="/") -# db = connect(host='', port=0, timeout=None, source_address=None) +# db = connect(host=''ort=0, timeout=None, source_address=None) instrumentator = Instrumentator().instrument(app) Instrumentator().instrument(app, metric_namespace='metro-api', metric_subsystem='metro-api').expose(app) @@ -703,7 +702,7 @@ async def get_all_routes(): @app.on_event("startup") async def startup_event(): - app.state.redis = await aioredis.from_url('redis://redis:6379') + app.state.redis = await aioredis.from_url(Config.REDIS_URL, socket_connect_timeout=5) await app.state.redis.flushdb() # Add this line to flush the Redis database uvicorn_access_logger = logging.getLogger("uvicorn.access") uvicorn_error_logger = logging.getLogger("uvicorn.error")