Merge pull request #1336 from cmu-delphi/release/delphi-epidata-4.1.13

Release Delphi Epidata 4.1.13

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.1.12
+current_version = 4.1.13
 commit = False
 tag = False
 

.git-blame-ignore-revs

@@ -1,7 +1,5 @@
 # style(black): format cdc acquisition
 980b0b7e80c7923b79e14fee620645e680785703
-# style(black): format covidcast_nowcast acquisition
-9e6ff16f599e8feec34a08dd1bddbc5eae347b55
 # style(black): format ecdc acquisition
 d1141d904da4e62992b97c92d5caebd8fadffd42
 # style(black): format flusurv acquisition

.github/workflows/performance-tests-one-time.yml

@@ -0,0 +1,106 @@
+name: One-time performance testing - 26th October 2023
+
+# Run "At every 30th minute on day-of-month 26 in October"
+on:
+  schedule:
+    - cron: '*/30 * 26 10 *'
+
+# Add some extra perms to comment on a PR
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  run-perftests:
+    # Run this on Delphi's self-hosted runner
+    runs-on: self-hosted
+    outputs:
+      request_count: ${{ steps.output.outputs.request_count }}
+      failure_count: ${{ steps.output.outputs.failure_count }}
+      med_time: ${{ steps.output.outputs.med_time }}
+      avg_time: ${{ steps.output.outputs.avg_time }}
+      min_time: ${{ steps.output.outputs.min_time }}
+      max_time: ${{ steps.output.outputs.max_time }}
+      requests_per_sec: ${{ steps.output.outputs.requests_per_sec }}
+    steps:
+      - name: Set up WireGuard
+        uses: egor-tensin/[email protected]
+        with:
+          endpoint: '${{ secrets.WG_PERF_ENDPOINT }}'
+          endpoint_public_key: '${{ secrets.WG_PERF_ENDPOINT_PUBLIC_KEY }}'
+          ips: '${{ secrets.WG_PERF_IPS }}'
+          allowed_ips: '${{ secrets.WG_PERF_ALLOWED_IPS }}'
+          private_key: '${{ secrets.WG_PERF_PRIVATE_KEY }}'
+      - name: Clean files from previous runs
+        uses: AutoModality/action-clean@v1
+      - name: Check out repository
+        uses: actions/checkout@v3
+      - name: Set up repository # mimics install.sh in the README except that delphi is cloned from the PR rather than main
+        run: |
+          cd ..
+          rm -rf driver
+          mkdir -p driver/repos/delphi
+          cd driver/repos/delphi
+          git clone https://github.com/cmu-delphi/operations
+          git clone https://github.com/cmu-delphi/utils
+          git clone https://github.com/cmu-delphi/flu-contest
+          git clone https://github.com/cmu-delphi/nowcast
+          cd ../../
+
+          cd ..
+          cp -R delphi-epidata driver/repos/delphi/delphi-epidata
+          cd -
+
+          ln -s repos/delphi/delphi-epidata/dev/local/Makefile
+      - name: Build & run epidata
+        run: |
+          cd ../driver
+          sudo make web sql="${{ secrets.DB_CONN_STRING }}" rate_limit="999999/second"
+          sudo make redis
+      - name: Check out delphi-admin
+        uses: actions/checkout@v3
+        with:
+          repository: cmu-delphi/delphi-admin
+          token: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+          path: delphi-admin
+      - name: Build & run Locust
+        continue-on-error: true # sometimes ~2-5 queries fail, we shouldn't end the run if that's the case
+        run: |
+          cd delphi-admin/load-testing/locust
+          docker build -t locust .
+          export CSV=v4-requests-small.csv
+          touch output_stats.csv && chmod 666 output_stats.csv
+          touch output_stats_history.csv && chmod 666 output_stats_history.csv
+          touch output_failures.csv && chmod 666 output_failures.csv
+          touch output_exceptions.csv && chmod 666 output_exceptions.csv
+          docker run --net=host -v $PWD:/mnt/locust -e CSV="/mnt/locust/${CSV}" locust -f /mnt/locust/v4.py --host http://127.0.0.1:10080/ --users 10 --spawn-rate 1 --headless -i "$(cat ${CSV} | wc -l)" --csv=/mnt/locust/output
+      - name: Produce output for summary
+        id: output
+        uses: jannekem/run-python-script-action@v1
+        with:
+          script: |
+            import os
+
+            def write_string(name, value):
+              with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+                print(f'{name}={value}', file=fh)
+
+            def write_float(name, value):
+              write_string(name, "{:.2f}".format(float(value)))
+
+            with open("delphi-admin/load-testing/locust/output_stats.csv", "r", encoding="utf-8", errors="ignore") as scraped:
+              final_line = scraped.readlines()[-1].split(",")
+              write_string('request_count', final_line[2])
+              write_string('failure_count', final_line[3])
+              write_float('med_time', final_line[4])
+              write_float('avg_time', final_line[5])
+              write_float('min_time', final_line[6])
+              write_float('max_time', final_line[7])
+              write_float('requests_per_sec', final_line[9])
+
+      - name: Archive results as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: locust-output
+          path: |
+            delphi-admin/load-testing/locust/output_*.csv

.github/workflows/performance-tests.yml

@@ -63,7 +63,7 @@ jobs:
       - name: Build & run epidata
         run: |
           cd ../driver
-          sudo make web sql="${{ secrets.DB_CONN_STRING }}"
+          sudo make web sql="${{ secrets.DB_CONN_STRING }}" rate_limit="999999/second"
           sudo make redis
       - name: Check out delphi-admin
         uses: actions/checkout@v3

deploy.json

@@ -208,15 +208,6 @@
       "add-header-comment": true
     },
 
-    "// acquisition - covidcast_nowcast",
-    {
-      "type": "move",
-      "src": "src/acquisition/covidcast_nowcast/",
-      "dst": "[[package]]/acquisition/covidcast_nowcast/",
-      "match": "^.*\\.(py)$",
-      "add-header-comment": true
-    },
-
     "// maintenance",
     {
       "type": "move",

dev/local/Makefile

@@ -37,6 +37,7 @@
 #   test=         Only runs tests in the directories provided here, e.g.
 #                 repos/delphi/delphi-epidata/tests/acquisition/covidcast
 #   sql=          Overrides the default SQL connection string.
+#   rate_limit=   Overrides the default rate limit for API requests.
 
 
 # Set optional argument defaults
@@ -56,6 +57,14 @@ else
 	sqlalchemy_uri:=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata
 endif
 
+ifdef rate_limit
+# Notation found here: https://flask-limiter.readthedocs.io/en/stable/#rate-limit-string-notation
+	rate_limit_settings:=--env "RATE_LIMIT=$(rate_limit)"
+else
+# Default behavior is to set the rate limit to "5/hour" for API key tests via this environment variable
+	rate_limit_settings:=--env "TESTING_MODE=True"
+endif
+
 SHELL:=/bin/sh
 
 # Get the Makefile's absolute path: https://stackoverflow.com/a/324782/4784655
@@ -104,7 +113,7 @@ web:
 		--env "REDIS_PASSWORD=1234" \
 		--env "API_KEY_ADMIN_PASSWORD=test_admin_password" \
 		--env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \
-		--env "TESTING_MODE=True" \
+		$(rate_limit_settings) \
 		--network delphi-net --name delphi_web_epidata \
 		delphi_web_epidata >$(LOG_WEB) 2>&1 &
 

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 4.1.12
+version = 4.1.13
 
 [options]
 packages =
@@ -13,7 +13,6 @@ packages =
     delphi.epidata.acquisition.covid_hosp.state_daily
     delphi.epidata.acquisition.covid_hosp.state_timeseries
     delphi.epidata.acquisition.covidcast
-    delphi.epidata.acquisition.covidcast_nowcast
     delphi.epidata.acquisition.ecdc
     delphi.epidata.acquisition.flusurv
     delphi.epidata.acquisition.fluview