Merge branch 'master' into stable

.circleci/config.yml

@@ -230,14 +230,14 @@ workflows:
             only: /.*/
     - rest-tests:
         requires:
-        - front-end-tests
+        - install-tator
         context: cvisionai
         filters:
           tags:
             only: /.*/
     - tator-py-tests:
         requires:
-        - front-end-tests
+        - install-tator
         context: cvisionai
         filters:
           tags:

.gitignore

@@ -40,6 +40,7 @@ Bento4-SDK-1-6-0-632.x86_64-unknown-linux/*
 google-chrome-stable_current_amd64.deb
 Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip
 .DS_Store
+*.DS_Store
 h
 .vscode/settings.json
 containers/tator_client_braw/Blackmagic_RAW_Linux_2.2.1.tar

containers/tator/Dockerfile

@@ -27,16 +27,16 @@ RUN pip3 --no-cache-dir --timeout=1000 install \
         psycopg2-binary==2.9.3 pillow==9.0.0 imageio==2.14.0 \
         pillow-avif-plugin==1.2.2 \
         djangorestframework==3.13.1 pygments==2.11.2 \
-        django-extensions==3.1.5 pygraphviz==1.8 \
+        django-extensions==3.1.5 pygraphviz==1.9 \
         pyparsing==3.0.7 pydot==1.4.2 markdown==3.3.6 \
-        hiredis==2.0.0 redis==4.1.1 greenlet==0.4.15 \
+        hiredis==2.0.0 redis==4.3.3 greenlet==0.4.15 \
         gunicorn==20.1.0 django_admin_json_editor==0.2.3 django-ltree==0.5.3 \
         requests==2.27.0 python-dateutil==2.8.2 ujson==5.1.0 slackclient==2.9.3 \
         google-auth==2.3.3 elasticsearch==7.10.1 progressbar2==4.0.0 \
         gevent==1.4.0 uritemplate==4.1.1 pylint pylint-django \
         django-cognito-jwt==0.0.4 boto3==1.20.41 \
         google-cloud-storage==2.1.0 datadog==0.43.0 \
-        kubernetes==21.7.0 minio==7.1.5
+        kubernetes==21.7.0 minio==7.1.5 okta-jwt-verifier==0.2.3
 
 # Get acme_tiny.py for certificate renewal
 WORKDIR /

helm/tator/Chart.yaml

@@ -9,11 +9,11 @@ sources:
   - https://github.com/cvisionai/tator
 dependencies:
   - name: metallb
-    version: 0.1.21
+    version: 3.0.7
     repository: https://charts.bitnami.com/bitnami
     condition: metallb.enabled
   - name: redis
-    version: 12.2.3
+    version: 16.11.3
     repository: https://charts.bitnami.com/bitnami
     condition: redis.enabled
   - name: elasticsearch
@@ -33,7 +33,7 @@ dependencies:
     repository: https://helm.min.io
     condition: minio.enabled
   - name: kube-prometheus-stack
-    version: 14.4.0
+    version: 36.2.0
     repository: https://prometheus-community.github.io/helm-charts
     condition: kube-prometheus-stack.enabled
   - name: prometheus-adapter

helm/tator/templates/_dbCron.tpl

@@ -1,5 +1,5 @@
 {{ define "dbCron.template" }}
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: {{ .name }}

helm/tator/templates/_nginx-server.tpl

@@ -73,6 +73,8 @@ server {
     {{- else }}
     add_header Cache-Control "max-age=0, must-revalidate";
     {{- end }}
+    add_header Cross-Origin-Opener-Policy same-origin;
+    add_header Cross-Origin-Embedder-Policy require-corp;
     {{- if hasKey .Values "allowCors" }}
     {{- if $.Values.allowCors }}
     add_header Access-Control-Allow-Origin *;
@@ -100,6 +102,8 @@ server {
     autoindex off;
     add_header Cache-Control "max-age=3600, must-revalidate";
     add_header 'Access-Control-Allow-Headers' 'Authorization' always;
+    add_header Cross-Origin-Opener-Policy same-origin;
+    add_header Cross-Origin-Embedder-Policy require-corp;
     
     {{- if hasKey .Values "allowCors" }}
     {{- if $.Values.allowCors }}
@@ -222,6 +226,8 @@ server {
     {{- if .Values.maintenance }}
     return 503;
     {{- end }}
+    add_header Cross-Origin-Opener-Policy same-origin;
+    add_header Cross-Origin-Embedder-Policy require-corp;
     proxy_pass http://gunicorn-svc:8000;
 
     proxy_redirect off;

helm/tator/templates/_tator.tpl

@@ -216,6 +216,22 @@ spec:
             {{- else }}
               value: "FALSE"
             {{- end }}
+            - name: OKTA_ENABLED
+            {{- if .Values.okta.enabled }}
+              value: "TRUE"
+            - name: OKTA_OAUTH2_KEY
+              value: {{ .Values.okta.oauth2_key }}
+            - name: OKTA_OAUTH2_SECRET
+              value: {{ .Values.okta.oauth2_secret }}
+            - name: OKTA_OAUTH2_TOKEN_URI
+              value: {{ .Values.okta.oauth2_token_uri }}
+            - name: OKTA_OAUTH2_ISSUER
+              value: {{ .Values.okta.oauth2_issuer }}
+            - name: OKTA_OAUTH2_AUTH_URI
+              value: {{ .Values.okta.oauth2_auth_uri }}
+            {{- else }}
+              value: "FALSE"
+            {{- end }}
             - name: ANONYMOUS_REGISTRATION_ENABLED
             {{- if hasKey .Values "anonymousRegistration" }}
             {{- if .Values.anonymousRegistration.enabled }}

helm/tator/templates/_tatorCron.tpl

@@ -1,5 +1,5 @@
 {{ define "tatorCron.template" }}
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: {{ .name }}

helm/tator/templates/pvc.yaml

@@ -21,7 +21,7 @@ spec:
     requests:
       storage: {{ .Values.pvc.size | default "10Ti" }}
 {{ end }}
-{{ if .Values.postgis.enabled }}
+{{ if hasKey .Values.postgis "hostPath" }}
 ---
 kind: PersistentVolumeClaim
 apiVersion: v1

helm/tator/values-aws.yaml

@@ -1,4 +1,4 @@
-domain: www.yourdomain.com
+domain: &domain www.yourdomain.com
 # These domains will work the same way as the primary domain.
 #aliases:
 #  - domain: alias.duckdns.org
@@ -246,6 +246,13 @@ remoteTranscodes:
     -----BEGIN CERTIFICATE-----
     <Insert certificate here>
     -----END CERTIFICATE-----
+okta:
+  enabled: false
+  oauth2_key: "fill me in"
+  oauth2_secret: "fill me in"
+  oauth2_token_uri: "fill me in"
+  oauth2_issuer: "fill me in"
+  oauth2_auth_uri: "fill me in"
 email:
   enabled: false
   sender: "<fill in>"

helm/tator/values-devExample.yaml

@@ -113,15 +113,15 @@ postgis:
 redis:
   # Enable this to install the redis helm chart.
   enabled: true
+  architecture: standalone
   master:
     persistence:
       enabled: false
-  slave:
-    persistence:
-      enabled: false
   nodeSelector:
     dbServer: "yes"
   usePassword: false
+  auth:
+    enabled: false
 metrics-server:
   enabled: true
   args:
@@ -284,6 +284,13 @@ cognito:
     aws-region: us-east-2
     pool-id: <POOL ID HERE>
     client-id: <ID HERE>
+okta:
+  enabled: false
+  oauth2_key: "fill me in"
+  oauth2_secret: "fill me in"
+  oauth2_token_uri: "fill me in"
+  oauth2_issuer: "fill me in"
+  oauth2_auth_uri: "fill me in"
 email:
   enabled: false
   sender: "<fill in>"

helm/tator/values-microk8s.yaml

@@ -133,15 +133,15 @@ postgis:
 redis:
   # Enable this to install the redis helm chart.
   enabled: true
+  architecture: standalone
   master:
     persistence:
       enabled: false
-  slave:
-    persistence:
-      enabled: false
   nodeSelector:
     dbServer: "yes"
   usePassword: false
+  auth:
+    enabled: false
 metrics-server:
   enabled: false
 elasticsearch:
@@ -227,6 +227,13 @@ cognito:
     aws-region: us-east-2
     pool-id: <POOL ID HERE>
     client-id: <ID HERE>
+okta:
+  enabled: false
+  oauth2_key: "fill me in"
+  oauth2_secret: "fill me in"
+  oauth2_token_uri: "fill me in"
+  oauth2_issuer: "fill me in"
+  oauth2_auth_uri: "fill me in"
 email:
   enabled: false
   sender: "<fill in>"

install.sh

@@ -6,13 +6,13 @@ set -e
 # Define environment variables.
 BENTO4_URL="http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip"
 GIT_REVISION=$(git rev-parse HEAD)
-KUBECTL_URL="https://storage.googleapis.com/kubernetes-release/release/v1.19.13/bin/linux/amd64/kubectl"
+KUBECTL_URL="https://dl.k8s.io/release/v1.22.9/bin/linux/amd64/kubectl"
 ARGO_CLIENT_URL="https://github.com/argoproj/argo-workflows/releases/download/v3.3.1/argo-linux-amd64.gz"
 ARGO_MANIFEST_URL="https://github.com/argoproj/argo-workflows/releases/download/v3.3.1/install.yaml"
 
 # Install snaps.
 sudo snap install helm --classic
-sudo snap install microk8s --classic --channel=1.19/stable
+sudo snap install microk8s --classic --channel=1.22/stable
 
 # Install apt packages.
 curl -sL https://deb.nodesource.com/setup_16.x | sudo -E bash -

main/models.py

@@ -245,32 +245,43 @@ def __str__(self):
         return self.name
 
 class TatorUserManager(UserManager):
-    def get_or_create_for_cognito(self, payload):
-        cognito_id = payload['sub']
+    valid_providers = ["cognito", "okta"]
+
+    def get_or_create_for_oauth2(self, payload, provider):
+        if provider not in self.valid_providers:
+            raise ValueError(
+                f"Expected oauth2 provider in {self.valid_providers}, got '{provider}'"
+            )
+
+        # The field to query in the user model is the provider name with `_id` appended
+        provider_id = {f"{provider}_id": payload["sub"]}
 
         try:
-            return self.get(cognito_id=cognito_id)
+            return self.get(**provider_id)
         except self.model.DoesNotExist:
             pass
 
-        first_name = payload['given_name']
-        last_name = payload['family_name']
+        first_name = payload["given_name"]
+        last_name = payload["family_name"]
         initials = f"{first_name[0]}{last_name[0]}"
-        user = User(
-            username=payload['email'],
-            cognito_id=cognito_id,
-            first_name=first_name,
-            last_name=last_name,
-            initials=initials,
-            email=payload['email'],
-            is_active=True)
+        user_definition = {
+            "username": payload["email"],
+            "first_name": first_name,
+            "last_name": last_name,
+            "initials": initials,
+            "email": payload["email"],
+            "is_active": True,
+        }
+        user_definition.update(provider_id)
+        user = User(**user_definition)
         user.save()
 
         return user
 
 class User(AbstractUser):
     objects=TatorUserManager()
-    cognito_id = UUIDField(primary_key=False,db_index=True,null=True,blank=True, editable=False)
+    cognito_id = UUIDField(primary_key=False, db_index=True, null=True, blank=True, editable=False)
+    okta_id = CharField(max_length=32, primary_key=False, db_index=True, null=True, blank=True, editable=False)
     middle_initial = CharField(max_length=1)
     initials = CharField(max_length=3)
     last_login = DateTimeField(null=True, blank=True)

main/rest/__init__.py

@@ -62,6 +62,7 @@
 from .membership import MembershipListAPI
 from .membership import MembershipDetailAPI
 from .notify import NotifyAPI
+from .oauth2 import Oauth2LoginAPI
 from .organization import OrganizationListAPI
 from .organization import OrganizationDetailAPI
 from .organization_upload_info import OrganizationUploadInfoAPI

main/rest/_jwt_helpers.py

@@ -0,0 +1,26 @@
+import asyncio
+import json
+
+from okta_jwt_verifier import AccessTokenVerifier, IDTokenVerifier
+
+
+loop = asyncio.new_event_loop()
+asyncio.set_event_loop(loop)
+
+
+def is_access_token_valid(token, issuer):
+    jwt_verifier = AccessTokenVerifier(issuer=issuer, audience="api://default")
+    try:
+        loop.run_until_complete(jwt_verifier.verify(token))
+        return True
+    except Exception:
+        return False
+
+
+def is_id_token_valid(token, issuer, client_id, nonce="SampleNonce"):
+    jwt_verifier = IDTokenVerifier(issuer=issuer, client_id=client_id, audience="api://default")
+    try:
+        loop.run_until_complete(jwt_verifier.verify(token, nonce=nonce))
+        return True
+    except Exception:
+        return False

main/rest/attribute_type.py

@@ -154,12 +154,19 @@ def _modify_attribute_type(cls, params: Dict, mod_type: str) -> Dict:
             # Determine if the attribute is being mutated
             attribute_mutated = False
             dtype_mutated = False
-            for key, new_value in new_attribute_type.items():
+
+            # Check all keys present in `new_attribute_type`
+            keys = set(new_attribute_type.keys())
+            if mod_type == "replace":
+                # Also check all keys present in `old_attribute_type` if this is a replacement
+                keys.update(old_attribute_type.keys())
+
+            for key in keys:
                 # Ignore differences in `name` values, those are handled by a rename
                 if key == "name":
                     continue
-                old_value = old_attribute_type.get(key)
-                if old_value is None or old_value != new_value:
+
+                if new_attribute_type.get(key) != old_attribute_type.get(key):
                     attribute_mutated = True
                     if key == "dtype":
                         dtype_mutated = True
@@ -231,7 +238,8 @@ def _modify_attribute_type(cls, params: Dict, mod_type: str) -> Dict:
                             # Mutate the entity attribute values
                             bulk_mutate_attributes(new_attribute, qs)
 
-                if mod_type == "PATCH":
+                if mod_type == "update":
+                    # An update is a combination of the new and old states
                     new_attribute_type = old_attribute_type.update(new_attribute_type)
 
                 messages.append(