Redesign Build Deployment Process (External)

.env

@@ -0,0 +1 @@
+DOCKER_IMAGE_TAG=2024-05-03--42-05

.github/fetch_runID.py

@@ -0,0 +1,44 @@
+
+import json
+import logging
+import requests
+import sys
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
+
+    '''
+        Workflow "docker image" uses image_build_push.yml
+        From above commented out code, and checked via terminal as well,
+        workflow id for the "docker image" can be fetched using:
+        https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows
+        https://api.github.com/repos/e-mission/e-mission-server/actions/workflows
+
+        For MukuFlash03: id = 75506902 
+        For e-mission-server: id = 35580278 
+    '''
+
+    download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs"
+    logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url)
+    r = requests.get(download_url)
+    if r.status_code != 200:
+        logging.debug(f"Unable to fetch workflow runs, status code: {r.status_code}")
+        sys.exit(1)
+    else:
+        workflow_runs_json = json.loads(r.text)
+        logging.debug(f"Successfully fetched workflow runs")
+
+        workflow_runs = workflow_runs_json["workflow_runs"]
+        if workflow_runs:
+            successful_runs = [run for run in workflow_runs \
+                                if run["status"] == "completed" and \
+                                run["conclusion"] == "success" and \
+                                run["head_branch"] == "master"
+                               ]
+            if successful_runs:
+                sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"])
+                sorted_runs = sorted(successful_runs, reverse=True, key=lambda x: x["updated_at"])
+                # print(sorted_runs)
+                latest_run_id = sorted_runs[0]["id"]
+                print(f"::set-output name=run_id::{latest_run_id}")
+

.github/workflows/image_build_push.yml

@@ -0,0 +1,141 @@
+name: docker-image-push-admin
+
+on:
+  push:
+    branches: [ master ]
+
+  workflow_dispatch:
+    inputs:
+      docker_image_tag:
+        description: "Latest Docker image tags passed from e-mission-server repository on image build and push"
+        required: true
+
+env:
+  DOCKER_USER: ${{secrets.DOCKER_USER}}
+  DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
+
+jobs:
+  fetch_run_id:
+    runs-on: ubuntu-latest
+
+    outputs:
+      run_id: ${{ steps.get_run_id.outputs.run_id }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with: 
+          python-version: '3.x'
+
+      - name: Install Python dependencies
+        run: |
+         python -m pip install --upgrade pip
+         pip install requests
+
+      - name: Run Python script
+        id: run_script
+        run: |
+          echo "Fetching latest successful run ID from e-misison-server docker image workflow"
+          python .github/fetch_runID.py
+
+      - name: Get Run ID
+        id: get_run_id
+        run: echo "run_id=${{ steps.run_script.outputs.run_id }}" >> "$GITHUB_OUTPUT"
+
+  fetch_tag:
+    needs: fetch_run_id
+    runs-on: ubuntu-latest
+
+    env:
+      RUN_ID: ${{ needs.fetch_run_id.outputs.run_id }}
+
+    outputs:
+      docker_image_tag: ${{ steps.get_docker_tag.outputs.docker_image_tag }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Use Run ID from previous fetch_run_id job
+      run: echo Run ID from previous job ${{ env.RUN_ID }}
+
+    - name: Download artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: docker-image-tag
+        # TODO: Create a token with basic repo permissions
+        github-token: ${{ secrets.GH_PAT_TAG }}
+        repository: e-mission/e-mission-server
+        run-id: ${{ env.RUN_ID }}
+
+    - name: Print artifact tag
+      id: get_docker_tag
+      run: |
+        cat tag_file.txt
+        docker_image_tag=$(cat tag_file.txt)
+        echo $docker_image_tag
+        echo "docker_image_tag=$(echo $docker_image_tag)" >> $GITHUB_OUTPUT
+
+  build:
+    needs: fetch_tag
+    runs-on: ubuntu-latest
+
+    env:
+      DOCKER_IMAGE_TAG_1: ${{ needs.fetch_tag.outputs.docker_image_tag }}
+      DOCKER_IMAGE_TAG_2: ${{ github.event.inputs.docker_image_tag }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Print input docker image tag
+      run: |
+        echo "Event name: ${{ github.event_name }}"
+        echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}"
+        echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}"
+
+    - name: Update .env file 
+      run: |
+        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env"
+          echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env
+        else
+          echo "Push event: Restoring latest server image tag in .env"
+          echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env
+        fi
+
+    - name: Add, Commit, Push changes to .env file
+      run: |
+        git config --local user.email "[email protected]"
+        git config --local user.name "GitHub Action"
+        if git diff --quiet; then
+          echo "Latest timestamp already present in .env file, no changes to commit"
+        else
+          git add .env
+          git commit -m "Updated docker image tag in .env file to the latest timestamp"
+          git push origin 
+        fi
+
+    - name: docker login
+      run: | # log into docker hub account
+        docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+
+    - name: Get current date # get the date of the build
+      id: date
+      run: echo "::set-output name=date::$(date +'%Y-%m-%d--%M-%S')"
+
+    - name: Run a one-line script
+      run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }}
+
+    - name: build docker image
+      run: |
+        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} .
+        else
+          docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} .
+        fi
+        docker images
+
+    - name: push docker image
+      run: |
+        docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }}

Dockerfile

@@ -0,0 +1,39 @@
+ARG DOCKER_IMAGE_TAG
+
+FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG}
+
+ENV DASH_DEBUG_MODE True
+ENV SERVER_PORT 8050
+
+# copy over setup files
+WORKDIR /usr/src/app/dashboard_setup
+COPY requirements.txt nrel_dash_components-0.0.1.tar.gz docker/setup.sh ./
+
+# install requirements.txt
+WORKDIR /usr/src/app
+RUN bash -c "./dashboard_setup/setup.sh"
+
+# copy over dashboard code
+WORKDIR /usr/src/app/pages
+COPY ./pages ./
+WORKDIR /usr/src/app/utils
+COPY ./utils ./
+WORKDIR /usr/src/app
+COPY app.py app_sidebar_collapsible.py globals.py globalsUpdater.py Procfile ./
+
+WORKDIR /usr/src/app/assets
+COPY assets/ ./
+RUN mkdir qrcodes
+
+# copy over test data
+WORKDIR /usr/src/app/data
+COPY data ./
+
+# open listening port, this may be overridden in docker-compose file
+EXPOSE ${SERVER_PORT}
+
+# run the dashboard
+WORKDIR /usr/src/app/dashboard_setup
+COPY docker/start.sh ./
+WORKDIR /usr/src/app
+CMD ["/bin/bash", "/usr/src/app/dashboard_setup/start.sh"]

app_sidebar_collapsible.py

@@ -17,6 +17,7 @@
 from dash import Input, Output, dcc, html, Dash
 import dash_auth
 import logging
+import base64
 # Set the logging right at the top to make sure that debug
 # logs are displayed in dev mode
 # until https://github.com/plotly/dash/issues/532 is fixed
@@ -29,15 +30,17 @@
 import flask_talisman as flt
 
 
-
-OPENPATH_LOGO = "https://www.nrel.gov/transportation/assets/images/openpath-logo.jpg"
+OPENPATH_LOGO = os.path.join(os.getcwd(), "assets/openpath-logo.jpg")
+encoded_image = base64.b64encode(open(OPENPATH_LOGO, 'rb').read()).decode("utf-8")
 auth_type = os.getenv('AUTH_TYPE')
 
 
 if auth_type == 'cognito':
     from utils.cognito_utils import authenticate_user, get_cognito_login_page
 elif auth_type == 'basic':
-    from config import VALID_USERNAME_PASSWORD_PAIRS
+    VALID_USERNAME_PASSWORD_PAIRS = {
+        'hello': 'world'
+    }
 
 app = Dash(
     external_stylesheets=[dbc.themes.BOOTSTRAP, dbc.icons.FONT_AWESOME],
@@ -59,7 +62,8 @@
             [
                 # width: 3rem ensures the logo is the exact width of the
                 # collapsed sidebar (accounting for padding)
-                html.Img(src=OPENPATH_LOGO, style={"width": "3rem"}),
+                # html.Img(src=OPENPATH_LOGO, style={"width": "3rem"}),
+                html.Img(src=f"data:image/png;base64,{encoded_image}", style={"width": "3rem"}), # Working
                 html.H2("OpenPATH"),
             ],
             className="sidebar-header",

docker-compose-dev.yml

@@ -5,6 +5,8 @@ services:
     build:
       context: .
       dockerfile: docker/Dockerfile
+      args:
+        DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG}
     image: e-mission/opdash:0.0.1
     ports:
       - "8050:8050"

docker-compose-prod-nginx.yml

@@ -6,6 +6,8 @@ services:
     build:
       context: .
       dockerfile: docker/Dockerfile
+      args:
+        DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG}
     image: e-mission/opdash:0.0.1
     environment:
       DASH_DEBUG_MODE: "True"

docker-compose-prod.yml

@@ -5,6 +5,8 @@ services:
     build:
       context: .
       dockerfile: docker/Dockerfile
+      args:
+        DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG}
     image: e-mission/opdash:0.0.1
     ports:
       - "8050:8050"

docker/Dockerfile

@@ -1,4 +1,6 @@
-FROM shankari/e-mission-server:master_2024-02-10--19-38
+ARG DOCKER_IMAGE_TAG
+
+FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG}
 
 ENV DASH_DEBUG_MODE True
 ENV SERVER_PORT 8050
@@ -17,10 +19,10 @@ COPY ./pages ./
 WORKDIR /usr/src/app/utils
 COPY ./utils ./
 WORKDIR /usr/src/app
-COPY app.py config.py app_sidebar_collapsible.py assets globals.py globalsUpdater.py Procfile ./
+COPY app.py app_sidebar_collapsible.py globals.py globalsUpdater.py Procfile ./
 
 WORKDIR /usr/src/app/assets
-COPY assets/style.css ./
+COPY assets/ ./
 RUN mkdir qrcodes
 
 # copy over test data

docker/start.sh

@@ -5,9 +5,8 @@ source setup/activate.sh
 echo "DB host = "${DB_HOST}
 if [ -z ${DB_HOST} ] ; then
     local_host=`hostname -i`
-    sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf
-else
-    sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf
+    export DB_HOST=$local_host
+    echo "Setting db host environment variable to localhost"
 fi
 
 # run the app

requirements.txt

@@ -1,14 +1,14 @@
 --extra-index-url https://plotly.nrel.gov/Docs/packages
 # dash is required to call `build:py`
-dash==2.9.3
+dash==2.16.1
 gunicorn==20.1.0
 plotly==5.14.1
 dash-bootstrap-components==1.4.1
 dash_extensions==0.1.13
 #dashboard_setup/nrel_dash_components-0.0.1.tar.gz # for docker-compose
 pylint==2.17.2
 qrcode==7.4.2
-pillow==10.0.1
+pillow==10.3.0
 requests==2.31.0
 python-jose==3.3.0
 flask==2.2.5

utils/cognito_utils.py

@@ -4,16 +4,16 @@
 import flask
 import requests
 import dash
+import os
 
-from config import CognitoConfig
 from utils import decode_jwt
 
 
 def get_tokens(code):
-    client_id = CognitoConfig.CLIENT_ID
-    client_secret = CognitoConfig.CLIENT_SECRET
-    redirect_uri = CognitoConfig.REDIRECT_URL
-    token_endpoint = CognitoConfig.TOKEN_ENDPOINT
+    client_id = os.getenv("COGNITO_CLIENT_ID", '')
+    client_secret = os.getenv("COGNITO_CLIENT_SECRET", '')
+    redirect_uri = os.getenv("COGNITO_REDIRECT_URL", '')
+    token_endpoint = os.getenv("COGNITO_TOKEN_ENDPOINT", '')
 
     encoded_data = base64.b64encode(f'{client_id}:{client_secret}'.encode('ascii')).decode('ascii')
     headers = {
@@ -59,7 +59,7 @@ def get_cognito_login_page(text='Welcome to the dashboard', color='black'):
                 dash.html.Label(text, style={
                     'font-size': '15px', 'display': 'block', 'verticalAlign': 'top', 'padding': '15px', 'color': color
                 }),
-                dbc.Button('Login with AWS Cognito', id='login-button', href=CognitoConfig.AUTH_URL, style={
+                dbc.Button('Login with AWS Cognito', id='login-button', href=os.getenv("COGNITO_AUTH_URL", ''), style={
                     'font-size': '14px', 'display': 'block', 'padding': '15px', 'verticalAlign': 'top',
                     'background-color': 'green', 'color': 'white'
                 }),