Update requirements and added separate docker dev and prod environments

slacgismo · Mar 14, 2024 · f58f4d6 · f58f4d6
1 parent 7ccc723
commit f58f4d6
Show file tree

Hide file tree

Showing 13 changed files with 187 additions and 27 deletions.
diff --git a/.gitignore b/.gitignore
@@ -148,4 +148,17 @@ ec2/time-shift-validation-hub
 ec2/test_analysis
 
 **/.vscode
-**/.venv
+**/.venv
+
+s3Emulator/pv-validation-hub-bucket/data_files/analytical/*
+s3Emulator/pv-validation-hub-bucket/data_files/ground_truth/*
+s3Emulator/pv-validation-hub-bucket/submission_files/*
+
+valhub/staticfiles
+valhub/media
+
+workers/tmp/*
+workers/current_evaluation/*
+
+
+*.pid
diff --git a/compressions/1/requirements.txt b/compressions/1/requirements.txt
@@ -1,2 +1,2 @@
-solardatatools
-numpy
+numpy
+solar-data-tools
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -15,6 +15,7 @@ services:
   api:
     build:
       context: ./valhub
+      dockerfile: Dockerfile
       args:
         djangosk: ${djangosk}
     env_file:
@@ -33,9 +34,15 @@ services:
     depends_on:
       - db
       - sqs
+    volumes:
+      - type: bind
+        source: ./valhub
+        target: /root/valhub
 
   s3:
-    build: ./s3Emulator
+    build:
+      context: ./s3Emulator
+      dockerfile: Dockerfile
     env_file:
       - .env
     environment:
@@ -44,9 +51,15 @@ services:
       - "5000:5000"
     depends_on:
       - api
+    volumes:
+      - type: bind
+        source: ./s3Emulator/pv-validation-hub-bucket
+        target: /pv-validation-hub-bucket
 
   ec2:
-    build: ./ec2
+    build:
+      context: ./ec2
+      dockerfile: Dockerfile
     env_file:
       - .env
     environment:
@@ -55,19 +68,29 @@ services:
       - "7000:7000"
     depends_on:
       - api
+    volumes:
+      - type: bind
+        source: ./ec2
+        target: /root/admin
 
   react-client:
     stdin_open: true
-    build: ./frontend
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile
     environment:
       HOST: 0.0.0.0
-    volumes:
-      - ./frontend:/opt/app:delegated
+    env_file:
+      - ./frontend/.env
     ports:
       - "3000:3000"
     depends_on:
       - api
       - s3
+    volumes:
+      - type: bind
+        source: ./frontend
+        target: /app
 
   sqs:
     image: softwaremill/elasticmq:latest
@@ -90,6 +113,9 @@ services:
       - api
       - s3
       - sqs
+    volumes:
+      - ./workers:/root/worker
+      - ./workers/tmp:/tmp
 
 volumes:
   valhub:
diff --git a/ec2/Dockerfile b/ec2/Dockerfile
@@ -16,6 +16,6 @@ RUN apt-get --assume-yes install mariadb-client supervisor postgresql-client
 RUN apt-get --assume-yes install python3-dev default-libmysqlclient-dev build-essential cmake git
 RUN python3 -m pip install --upgrade pip
 RUN python3 -m pip install -r requirements.txt
-RUN git clone https://github.com/kperrynrel/time-shift-validation-hub.git
+# RUN git clone https://github.com/kperrynrel/time-shift-validation-hub.git
 
 ENTRYPOINT ["tail", "-f", "/dev/null"]
diff --git a/ec2/requirements.txt b/ec2/requirements.txt
@@ -1,5 +1,5 @@
-numpy==1.22.0
-pandas==1.5.3
+numpy
+pandas
 boto3
 
 ########## add analytical task/evaluation script dependcies here to be pre-installed on the runner.

diff --git a/frontend/Dockerfile b/frontend/Dockerfile
@@ -2,9 +2,5 @@ FROM node:16-alpine as builder
 WORKDIR /app
 COPY package.json .
 RUN npm install
-COPY . .
-COPY ./.env.example .env
-RUN npm run build
-RUN npm install -g serve
 EXPOSE 3000
-CMD ["npx", "serve", "-s", "build"]
+CMD ["npm", "start"]
diff --git a/frontend/prod.Dockerfile b/frontend/prod.Dockerfile
@@ -0,0 +1,10 @@
+FROM node:16-alpine as builder
+WORKDIR /app
+COPY package.json .
+RUN npm install
+COPY . .
+COPY ./.env.example .env
+RUN npm run build
+RUN npm install -g serve
+EXPOSE 3000
+CMD ["npx", "serve", "-s", "build"]
diff --git a/prod.docker-compose.yml b/prod.docker-compose.yml
@@ -0,0 +1,103 @@
+version: "3.8"
+
+services:
+  db:
+    image: postgres:15-alpine
+    environment:
+      POSTGRES_DB: valhub
+      POSTGRES_PASSWORD: valhub
+      POSTGRES_USER: valhub
+    ports:
+      - "5432:5432"
+    volumes:
+      - valhub:/data/db
+
+  api:
+    build:
+      context: ./valhub
+      dockerfile: prod.Dockerfile
+      args:
+        djangosk: ${djangosk}
+    env_file:
+      - .env
+    environment:
+      PGDATABASE: valhub
+      PGUSER: valhub
+      PGHOST: db
+      PG_DATABASE_URI: "postgresql://valhub:valhub@db/valhub"
+      DJANGO_SUPERUSER_USERNAME: ${admin_username}
+      DJANGO_SUPERUSER_PASSWORD: ${admin_password}
+      DJANGO_SUPERUSER_EMAIL: ${admin_email}
+      DJANGO_SECRET_KEY: ${djangosk}
+    ports:
+      - "8005:8005"
+    depends_on:
+      - db
+      - sqs
+
+  s3:
+    build:
+      context: ./s3Emulator
+      dockerfile: prod.Dockerfile
+    env_file:
+      - .env
+    environment:
+      HOST: 0.0.0.0
+    ports:
+      - "5000:5000"
+    depends_on:
+      - api
+
+  ec2:
+    build:
+      context: ./ec2
+      dockerfile: prod.Dockerfile
+    env_file:
+      - .env
+    environment:
+      HOST: 0.0.0.0
+    ports:
+      - "7000:7000"
+    depends_on:
+      - api
+
+  react-client:
+    stdin_open: true
+    build:
+      context: ./frontend
+      dockerfile: prod.Dockerfile
+    environment:
+      HOST: 0.0.0.0
+    volumes:
+      - ./frontend:/opt/app:delegated
+    ports:
+      - "3000:3000"
+    depends_on:
+      - api
+      - s3
+
+  sqs:
+    image: softwaremill/elasticmq:latest
+    ports:
+      - "9324:9324"
+    environment:
+      - "ELASTICMQ_OPTS=-Dconfig.file=/opt/elasticmq.conf"
+    volumes:
+      - ./custom.conf:/opt/elasticmq.conf
+      - ./application.ini:/opt/docker/conf/application.ini
+
+  worker:
+    build:
+      context: ./workers
+      dockerfile: prod.Dockerfile
+    env_file:
+      - .env
+    ports:
+      - "8500:8500"
+    depends_on:
+      - api
+      - s3
+      - sqs
+
+volumes:
+  valhub:
diff --git a/s3Emulator/Dockerfile b/s3Emulator/Dockerfile
@@ -4,7 +4,7 @@ FROM python:3.9-slim-buster
 WORKDIR /
 
 RUN apt-get update && apt-get install git sudo curl jq gawk nano -y
-RUN git clone https://github.com/kperrynrel/time-shift-validation-hub.git
+# RUN git clone https://github.com/kperrynrel/time-shift-validation-hub.git
 
 # Set the working directory
 WORKDIR /app
@@ -26,9 +26,17 @@ WORKDIR /app
 RUN pip install -r requirements.txt
 
 # Move simulated bucket to location 
-RUN mv ./pv-validation-hub-bucket /
-RUN mv /time-shift-validation-hub/data/file_data/* /pv-validation-hub-bucket/data_files/analytical/
-RUN mv /time-shift-validation-hub/data/validation_data/* /pv-validation-hub-bucket/data_files/ground_truth/
+# RUN mv ./pv-validation-hub-bucket /
+# RUN mv /time-shift-validation-hub/data/file_data/* /pv-validation-hub-bucket/data_files/analytical/
+# RUN mv /time-shift-validation-hub/data/validation_data/* /pv-validation-hub-bucket/data_files/ground_truth/
+
+# Make simulated bucket in location
+RUN mkdir /pv-validation-hub-bucket
+RUN mkdir /pv-validation-hub-bucket/data_files
+RUN mkdir /pv-validation-hub-bucket/data_files/analytical
+RUN mkdir /pv-validation-hub-bucket/data_files/ground_truth
+RUN mkdir /pv-validation-hub-bucket/submission_files
+
 # Install the required dependencies
 RUN pip install flask boto3
 

diff --git a/valhub/Dockerfile b/valhub/Dockerfile
@@ -6,8 +6,12 @@ WORKDIR /root
 # COPY .aws /root/.aws
 RUN mkdir valhub
 WORKDIR /root/valhub
+
+VOLUME /root/valhub
+
 COPY . .
 
+
 # Set Django admin username, password, and other environment variables
 ARG admin_username
 ARG admin_password
@@ -29,7 +33,7 @@ RUN python3 -m pip install --upgrade pip
 WORKDIR /root/valhub
 RUN python3 -m pip install -r requirements.txt
 
-RUN cd ./valhub
+# RUN cd ./valhub
 RUN python3 manage.py makemigrations
-RUN python3 manage.py collectstatic
+RUN python3 manage.py collectstatic --noinput
 CMD ["/usr/bin/supervisord", "-c", "/root/valhub/supervisord.conf"]
diff --git a/valhub/supervisord.conf b/valhub/supervisord.conf
@@ -3,7 +3,7 @@ nodaemon = true
 
 [program:valhub_webserver]
 directory=/root/valhub
-command=python3 /root/valhub/manage.py runserver --noreload 0.0.0.0:8005
+command=python3 /root/valhub/manage.py runserver 0.0.0.0:8005
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 redirect_stderr=true

diff --git a/workers/Dockerfile b/workers/Dockerfile
@@ -19,6 +19,6 @@ WORKDIR /root
 WORKDIR /root/worker
 RUN python3 -m pip install --upgrade pip
 WORKDIR /root/worker
-RUN python3 -m pip install -r requirements.txt
+RUN python3 -m pip install -r requirements.txt --timeout=1000
 
 CMD ["/bin/sh", "/root/worker/docker-entrypoint.sh"]
diff --git a/workers/requirements.txt b/workers/requirements.txt
@@ -1,6 +1,6 @@
-matplotlib==3.7.1
-numpy==1.22.0
-pandas==1.5.3
+matplotlib
+numpy
+pandas
 pvanalytics==0.1.3
 pvlib==0.9.4
 ruptures==1.1.7