Updated chart data, S3 aggregate parquet moved (#124)

* Updated chart data, S3 aggregate parquet moved

* PR feedback

* updated api spec

docs/dashboard_api.prod.yaml

@@ -8,13 +8,8 @@ servers:
     basePath:
       default: "/"
 paths:
-  /data_packages:
+  /metadata:
     get:
-      parameters:
-      - name: "name"
-        in: "query"
-        schema:
-          type: "string"
       security:
       - api_key: []
     options:
@@ -35,15 +30,14 @@ paths:
           content: {}
       security:
       - api_key: []
-  /metadata:
+  /last-valid:
     get:
       security:
       - api_key: []
     options:
-      summary: "CORS support"
       responses:
         "200":
-          description: "Default response for CORS method"
+          description: "200 response"
           headers:
             Access-Control-Allow-Origin:
               schema:
@@ -57,27 +51,35 @@ paths:
           content: {}
       security:
       - api_key: []
-  /chart-data/{subscription_name}:
+  /data-packages/{data_package_id}/chart:
     get:
       parameters:
-      - name: "subscription_name"
+      - name: "data_package_id"
         in: "path"
         required: true
         schema:
           type: "string"
+      - name: "column"
+        in: "query"
+        required: true
+        schema:
+          type: "string"
+      - name: "filters"
+        in: "query"
+        schema:
+          type: "string"
       security:
       - api_key: []
     options:
-      summary: "CORS support"
       parameters:
-      - name: "subscription_name"
+      - name: "data_package_id"
         in: "path"
         required: true
         schema:
           type: "string"
       responses:
         "200":
-          description: "Default response for CORS method"
+          description: "200 response"
           headers:
             Access-Control-Allow-Origin:
               schema:
@@ -113,28 +115,6 @@ paths:
           content: {}
       security:
       - api_key: []
-  /last_valid:
-    get:
-      security:
-      - api_key: []
-    options:
-      summary: "CORS support"
-      responses:
-        "200":
-          description: "Default response for CORS method"
-          headers:
-            Access-Control-Allow-Origin:
-              schema:
-                type: "string"
-            Access-Control-Allow-Methods:
-              schema:
-                type: "string"
-            Access-Control-Allow-Headers:
-              schema:
-                type: "string"
-          content: {}
-      security:
-      - api_key: []
   /metadata/{site}/{study}:
     get:
       parameters:
@@ -351,7 +331,33 @@ paths:
           content: {}
       security:
       - api_key: []
-  /last_valid/{study}/{data_package}/{site}/{version}/{filename}:
+  /data-packages:
+    get:
+      parameters:
+      - name: "name"
+        in: "query"
+        schema:
+          type: "string"
+      security:
+      - api_key: []
+    options:
+      responses:
+        "200":
+          description: "200 response"
+          headers:
+            Access-Control-Allow-Origin:
+              schema:
+                type: "string"
+            Access-Control-Allow-Methods:
+              schema:
+                type: "string"
+            Access-Control-Allow-Headers:
+              schema:
+                type: "string"
+          content: {}
+      security:
+      - api_key: []
+  /last-valid/{study}/{data_package}/{site}/{version}/{filename}:
     get:
       parameters:
       - name: "filename"
@@ -382,7 +388,6 @@ paths:
       security:
       - api_key: []
     options:
-      summary: "CORS support"
       parameters:
       - name: "filename"
         in: "path"
@@ -411,7 +416,7 @@ paths:
           type: "string"
       responses:
         "200":
-          description: "Default response for CORS method"
+          description: "200 response"
           headers:
             Access-Control-Allow-Origin:
               schema:
@@ -425,27 +430,26 @@ paths:
           content: {}
       security:
       - api_key: []
-  /study-periods/{site}:
+  /data-packages/{data_package_id}:
     get:
       parameters:
-      - name: "site"
+      - name: "data_package_id"
         in: "path"
         required: true
         schema:
           type: "string"
       security:
       - api_key: []
     options:
-      summary: "CORS support"
       parameters:
-      - name: "site"
+      - name: "data_package_id"
         in: "path"
         required: true
         schema:
           type: "string"
       responses:
         "200":
-          description: "Default response for CORS method"
+          description: "200 response"
           headers:
             Access-Control-Allow-Origin:
               schema:
@@ -459,19 +463,9 @@ paths:
           content: {}
       security:
       - api_key: []
-  /metadata/{site}/{study}/{data_package}:
+  /study-periods/{site}:
     get:
       parameters:
-      - name: "data_package"
-        in: "path"
-        required: true
-        schema:
-          type: "string"
-      - name: "study"
-        in: "path"
-        required: true
-        schema:
-          type: "string"
       - name: "site"
         in: "path"
         required: true
@@ -482,16 +476,6 @@ paths:
     options:
       summary: "CORS support"
       parameters:
-      - name: "data_package"
-        in: "path"
-        required: true
-        schema:
-          type: "string"
-      - name: "study"
-        in: "path"
-        required: true
-        schema:
-          type: "string"
       - name: "site"
         in: "path"
         required: true
@@ -513,19 +497,40 @@ paths:
           content: {}
       security:
       - api_key: []
-  /data_packages/{id}:
+  /metadata/{site}/{study}/{data_package}:
     get:
       parameters:
-      - name: "id"
+      - name: "data_package"
+        in: "path"
+        required: true
+        schema:
+          type: "string"
+      - name: "study"
+        in: "path"
+        required: true
+        schema:
+          type: "string"
+      - name: "site"
         in: "path"
         required: true
         schema:
           type: "string"
       security:
       - api_key: []
     options:
+      summary: "CORS support"
       parameters:
-      - name: "id"
+      - name: "data_package"
+        in: "path"
+        required: true
+        schema:
+          type: "string"
+      - name: "study"
+        in: "path"
+        required: true
+        schema:
+          type: "string"
+      - name: "site"
         in: "path"
         required: true
         schema:

scripts/migrations/migration.004.s3_name_with_id.py

@@ -0,0 +1,47 @@
+"""Removes unexpected root nodes/templates/misspelled keys from transaction log."""
+
+import argparse
+import io
+import json
+
+import boto3
+
+
+def _get_s3_data(key: str, bucket_name: str, client) -> dict:
+    """Convenience class for retrieving a dict from S3"""
+    try:
+        bytes_buffer = io.BytesIO()
+        client.download_fileobj(Bucket=bucket_name, Key=key, Fileobj=bytes_buffer)
+        return json.loads(bytes_buffer.getvalue().decode())
+    except Exception:  # pylint: disable=broad-except
+        return {}
+
+
+def _put_s3_data(key: str, bucket_name: str, client, data: dict) -> None:
+    """Convenience class for writing a dict to S3"""
+    b_data = io.BytesIO(json.dumps(data).encode())
+    client.upload_fileobj(Bucket=bucket_name, Key=key, Fileobj=b_data)
+
+
+def s3_name_with_id(bucket: str):
+    client = boto3.client("s3")
+    res = client.list_objects_v2(Bucket=bucket)
+    contents = res["Contents"]
+    moved_files = 0
+    for s3_file in contents:
+        key = s3_file["Key"]
+        key_array = key.split("/")
+        if key_array[0] == "aggregates" and len(key_array[3]) == 3:
+            key_array[3] = f"{key_array[2]}__{key_array[3]}"
+            new_key = "/".join(key_array)
+            client.copy({"Bucket": bucket, "Key": key}, bucket, new_key)
+            client.delete_object(Bucket=bucket, Key=key)
+            moved_files += 1
+    print(f"Updated {moved_files} aggregates")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="""Changes lowest directory in S3 to file id""")
+    parser.add_argument("-b", "--bucket", help="bucket name")
+    args = parser.parse_args()
+    s3_name_with_id(args.bucket)