nmfs-openscapes: make adjustments to infra

config/clusters/nmfs-openscapes/enc-deployer-credentials.secret.json

@@ -0,0 +1,25 @@
+{
+    "AccessKey": {
+        "AccessKeyId": "ENC[AES256_GCM,data:97VM/eSCAMnEWvcj2pOBTVdCI/M=,iv:yaQfp5fGzJgbbkbSqbGyUnn49Y0zBdUf6qj2MQUe5e4=,tag:tGqMBMbsL7VMT8PKN3e2fA==,type:str]",
+        "SecretAccessKey": "ENC[AES256_GCM,data:IzxyWJNvqI4lFzuPbC/b+3w2eSWAuDk58cyi7bU0EPxQ1TefK/LTtg==,iv:z28cJYV/066dtpInIJ2fpwO/8ti8o98YpAZVEnFYlPQ=,tag:Df/tLa6dwXvvf3XWDndtuQ==,type:str]",
+        "UserName": "ENC[AES256_GCM,data:afMU/OOXogtbb3XDrtUb5j0Ij3qydWI=,iv:l1Dh459VjD53M1hnhqyhY16w2c4J+FgOpjeuENqQkGc=,tag:ClWtdIEfTjAolwiQZD14CQ==,type:str]"
+    },
+    "sops": {
+        "kms": null,
+        "gcp_kms": [
+            {
+                "resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
+                "created_at": "2024-09-09T14:44:20Z",
+                "enc": "CiUA4OM7eNCK+1vrOfGfoiorEq68Zyw0ttYDYOcV1aO5r5RFEmBDEkkA5dG1Q09E32GJL/EniI5GgKcdjY4WmD02cPHKkYWF/kyvc7OIGnyYqDm53nLl/ITHs/gi0TroovJ9xpgKTomZfQwY5qA8pv6S"
+            }
+        ],
+        "azure_kv": null,
+        "hc_vault": null,
+        "age": null,
+        "lastmodified": "2024-09-09T14:44:21Z",
+        "mac": "ENC[AES256_GCM,data:+vDQFVgWJV5dGUw/tyUdRL55w/Amx7wXCPxK7em7iI2JQqWdPZ0Qv8qhz5RyIXVLlzgQuFkzRR2jvXZ3G8kIT0MQsmx5Vcnf6+MEQrxroFnG1avvQkBXEzIHkMYgDoBAM+zb+mauq3mFNGg3antiVJ9shmlBlnMBkwoDSewBWz8=,iv:vgjDRoAMVu59Rq+gFQNK2Ty5mM7XQ2wGf6PysPqqI9M=,tag:di/pmrQxkhxP5IGNznUtLw==,type:str]",
+        "pgp": null,
+        "unencrypted_suffix": "_unencrypted",
+        "version": "3.9.0"
+    }
+}

eksctl/nmfs-openscapes.jsonnet

@@ -25,9 +25,42 @@ local nodeAz = "us-west-2b";
 // A `node.kubernetes.io/instance-type label is added, so pods
 // can request a particular kind of node with a nodeSelector
 local notebookNodes = [
-    { instanceType: "r7i.xlarge" },
-    { instanceType: "r7i.4xlarge" },
-    { instanceType: "r7i.16xlarge" },
+    {
+        instanceType: "r7i.xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c.org/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    {
+        instanceType: "r7i.4xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c.org/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    {
+        instanceType: "r7i.16xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c.org/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    {
+        instanceType: "r7i.xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c.org/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+    },
+    {
+        instanceType: "r7i.4xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c.org/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+    },
+    {
+        instanceType: "r7i.16xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c.org/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+    },
 ];
 local daskNodes = [];
 
@@ -53,24 +86,31 @@ local daskNodes = [];
     //    eksctl create addon --config-file=nmfs-openscapes.eksctl.yaml
     //
     addons: [
-        {
-            // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
-            // couple to AWS EBS based storage, without it expect to see pods
-            // mounting a PVC failing to schedule and PVC resources that are
-            // unbound.
-            //
-            // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
-            //
-            name: 'aws-ebs-csi-driver',
-            version: "latest",
-            wellKnownPolicies: {
-                ebsCSIController: true,
+        { version: "latest", tags: $.metadata.tags } + addon
+        for addon in
+        [
+            {
+                name: "vpc-cni",
+                configurationValues: |||
+                    enableNetworkPolicy: "true"
+                |||
             },
-            tags: {
-                "ManagedBy": "2i2c",
-                "2i2c.org/cluster-name": $.metadata.name,
+            { name: "coredns" },
+            { name: "kube-proxy" },
+            {
+                // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
+                // couple to AWS EBS based storage, without it expect to see pods
+                // mounting a PVC failing to schedule and PVC resources that are
+                // unbound.
+                //
+                // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
+                //
+                name: "aws-ebs-csi-driver",
+                wellKnownPolicies: {
+                    ebsCSIController: true,
+                },
             },
-        },
+        ]
     ],
     nodeGroups: [
     n + {clusterName: $.metadata.name} for n in
@@ -83,7 +123,7 @@ local daskNodes = [];
             ssh: {
                 publicKeyPath: 'ssh-keys/nmfs-openscapes.key.pub'
             },
-            instanceType: "r5.xlarge",
+            instanceType: "r7i.xlarge",
             minSize: 1,
             maxSize: 6,
             labels+: {
@@ -136,4 +176,4 @@ local daskNodes = [];
         ] else []
     )
     ]
-}
+}

terraform/aws/projects/nmfs-openscapes.tfvars

@@ -1,30 +1,45 @@
-/*
- Some of the assumptions this jinja2 template makes about the cluster:
-   - location of the nodes of the kubernetes cluster will be <region>a
-   - no default scratch buckets support
-*/
 region                 = "us-west-2"
 cluster_name           = "nmfs-openscapes"
 cluster_nodes_location = "us-west-2b"
 
-# Tip: uncomment and fill the missing info in the lines below if you want
-#       to setup scratch buckets for the hubs on this cluster.
-#
-#user_buckets = {
-#  "scratch-staging" : {
-#    "delete_after" : 7,
-#  },
-#  # Tip: add more scratch buckets below, if this cluster will be multi-tenant
-#}
+disable_cluster_wide_filestore = true
+filestores = {
+  "staging" = {
+    name_suffix = "staging",
+    tags        = { "2i2c:hub-name" : "staging" },
+  },
+  "prod" = {
+    name_suffix = "prod",
+    tags        = { "2i2c:hub-name" : "prod" },
+  },
+}
 
-# Tip: uncomment and fill the missing info in the lines below if you want
-#       to setup specific cloud permissions for the buckets in this cluster.
-#
-#hub_cloud_permissions = {
-#  "staging" : {
-#    "user-sa" : {
-#      bucket_admin_access : ["scratch-staging"],
-#    },
-#  },
-#  # Tip: add more namespaces below, if this cluster will be multi-tenant
-#}
+user_buckets = {
+  "scratch-staging" : {
+    "delete_after" : 7,
+    "tags" : { "2i2c:hub-name" : "staging" },
+  },
+  "scratch" : {
+    "delete_after" : 7,
+    "tags" : { "2i2c:hub-name" : "prod" },
+  },
+  "persistent-staging" : {
+    "tags" : { "2i2c:hub-name" : "staging" },
+  },
+  "persistent" : {
+    "tags" : { "2i2c:hub-name" : "prod" },
+  },
+}
+
+hub_cloud_permissions = {
+  "staging" : {
+    "user-sa" : {
+      bucket_admin_access : ["scratch-staging", "persistent-staging"],
+    },
+  },
+  "prod" : {
+    "user-sa" : {
+      bucket_admin_access : ["scratch", "persistent"],
+    },
+  },
+}