From 1f542610e6ea3a6d536463767d0fd690ac1759c3 Mon Sep 17 00:00:00 2001 From: iameskild Date: Wed, 31 May 2023 12:05:34 -0700 Subject: [PATCH 01/12] Add ARGO env vars to user profile, update jupyter_server_config name --- .../services/jupyterhub/configmaps.tf | 10 ++--- ...ig.py.tpl => jupyter_server_config.py.tpl} | 0 .../files/jupyterhub/03-profiles.py | 40 ++++++++++++++++++- .../kubernetes/services/jupyterhub/main.tf | 1 + 4 files changed, 45 insertions(+), 6 deletions(-) rename src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/{jupyter_notebook_config.py.tpl => jupyter_server_config.py.tpl} (100%) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/configmaps.tf b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/configmaps.tf index 41628d3241..4f8c38464d 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/configmaps.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/configmaps.tf @@ -1,5 +1,5 @@ locals { - jupyter-notebook-config-py-template = templatefile("${path.module}/files/jupyter/jupyter_notebook_config.py.tpl", { + jupyter-notebook-config-py-template = templatefile("${path.module}/files/jupyter/jupyter_server_config.py.tpl", { terminal_cull_inactive_timeout = var.idle-culler-settings.terminal_cull_inactive_timeout terminal_cull_interval = var.idle-culler-settings.terminal_cull_interval kernel_cull_idle_timeout = var.idle-culler-settings.kernel_cull_idle_timeout @@ -12,9 +12,9 @@ locals { } -resource "local_file" "jupyter_notebook_config_py" { +resource "local_file" "jupyter_server_config_py" { content = local.jupyter-notebook-config-py-template - filename = "${path.module}/files/jupyter/jupyter_notebook_config.py" + filename = "${path.module}/files/jupyter/jupyter_server_config.py" } @@ -33,7 +33,7 @@ resource "kubernetes_config_map" "etc-ipython" { resource "kubernetes_config_map" "etc-jupyter" { depends_on = [ - local_file.jupyter_notebook_config_py + local_file.jupyter_server_config_py ] metadata { @@ -42,7 +42,7 @@ resource "kubernetes_config_map" "etc-jupyter" { } data = { - "jupyter_notebook_config.py" : local_file.jupyter_notebook_config_py.content + "jupyter_server_config.py" : local_file.jupyter_server_config_py.content } } diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_notebook_config.py.tpl b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl similarity index 100% rename from src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_notebook_config.py.tpl rename to src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index 255d5e1fe0..4c387d3c5f 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -311,6 +311,40 @@ def configure_user(username, groups, uid=1000, gid=100): } +def profile_argo_token(groups): + # TODO: create a more robust check user's Argo-Workflow role + + domain = z2jh.get_config("custom.external-url") + + ADMIN = "admin" + DEVELOPER = "developer" + ANALYST = "analyst" + + base = "argo-" + + if ANALYST in groups: + argo_sa = base + "viewer" + if DEVELOPER in groups: + argo_sa = base + "developer" + if ADMIN in groups: + argo_sa = base + "admin" + else: + return {} + + return { + "ARGO_BASE_HREF": "/argo", + "ARGO_SERVER": f"{domain}:443", + "ARGO_TOKEN": { + "valueFrom": { + "secretKeyRef": { + "name": f"{argo_sa}.service-account-token", + "key": "token", + } + } + }, + } + + def render_profile(profile, username, groups, keycloak_profilenames): """Render each profile for user. @@ -366,7 +400,7 @@ def render_profile(profile, username, groups, keycloak_profilenames): def preserve_envvars(spawner): # This adds in JUPYTERHUB_ANYONE/GROUP rather than overwrite all env vars, # if set in the spawner for a dashboard to control access. - return {**envvars_fixed, **spawner.environment} + return {**envvars_fixed, **spawner.environment, **profile_argo_token(groups)} profile["kubespawner_override"]["environment"] = preserve_envvars @@ -404,6 +438,10 @@ def render_profiles(spawner): ) +c.KubeSpawner.args = ["--debug"] +c.KubeSpawner.environment = { + "JUPYTERHUB_SINGLEUSER_APP": "jupyter_server.serverapp.ServerApp", +} c.KubeSpawner.profile_list = render_profiles diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf index 4d2ced25cc..00702a871d 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf @@ -24,6 +24,7 @@ resource "helm_release" "jupyterhub" { jsonencode({ # custom values can be accessed via z2jh.get_config('custom.') custom = { + external-url = var.external-url theme = var.theme profiles = var.profiles cdsdashboards = var.cdsdashboards From a1927b9d4601898f54193fea55dfa980d9e65c4c Mon Sep 17 00:00:00 2001 From: iameskild Date: Thu, 1 Jun 2023 13:56:12 -0700 Subject: [PATCH 02/12] Add auth-mode=client, update jupyter_server_config --- .../modules/kubernetes/services/argo-workflows/main.tf | 2 +- .../jupyterhub/files/jupyter/jupyter_server_config.py.tpl | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf index a7182e6a37..565e85c598 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf @@ -30,7 +30,7 @@ resource "helm_release" "argo-workflows" { server = { # `sso` for OIDC/OAuth - extraArgs = ["--auth-mode=sso", "--insecure-skip-verify"] + extraArgs = ["--auth-mode=sso", "--auth-mode=client", "--insecure-skip-verify"] # to enable TLS, `secure = true` secure = false baseHref = "/${local.argo-workflows-prefix}/" diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl index c33dbe4f3c..1d1aac70b4 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl @@ -36,3 +36,8 @@ c.NotebookApp.shutdown_no_activity_timeout = ${server_shutdown_no_activity_timeo ############################################################################### # JupyterHub idle culler total timeout corresponds (approximately) to: # max(cull_idle_timeout, cull_inactive_timeout) + shutdown_no_activity_timeout + +from argo_workflows_executor import ArgoExecutor, ArgoScheduler + +c.Scheduler.execution_manager_class=ArgoExecutor +c.SchedulerApp.scheduler_class=ArgoScheduler From a74a3d292f71555b3cb34a33dca8730eb7d24352 Mon Sep 17 00:00:00 2001 From: iameskild Date: Thu, 1 Jun 2023 21:41:05 -0700 Subject: [PATCH 03/12] Use conda-store API to validate conda env used --- src/_nebari/stages/input_vars.py | 6 ++++ .../07-kubernetes-services/jupyterhub.tf | 13 +++++---- .../jupyter/jupyter_server_config.py.tpl | 1 + .../files/jupyterhub/03-profiles.py | 28 ++++++++++++++++++- .../kubernetes/services/jupyterhub/main.tf | 16 +++++++++++ .../services/jupyterhub/variables.tf | 5 ++++ 6 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/_nebari/stages/input_vars.py b/src/_nebari/stages/input_vars.py index 85859cd2b9..eff4e71daa 100644 --- a/src/_nebari/stages/input_vars.py +++ b/src/_nebari/stages/input_vars.py @@ -300,6 +300,12 @@ def stage_07_kubernetes_services(stage_outputs, config): "*/*": ["viewer"], }, }, + "argo-workflows-jupyter-scheduler": { + "primary_namespace": "", + "role_bindings": { + "*/*": ["viewer"], + }, + }, }, "conda-store-default-namespace": config.get("conda_store", {}).get( "default_namespace", "nebari-git" diff --git a/src/_nebari/template/stages/07-kubernetes-services/jupyterhub.tf b/src/_nebari/template/stages/07-kubernetes-services/jupyterhub.tf index 25875e62cc..ed3a478d7b 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/jupyterhub.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/jupyterhub.tf @@ -98,12 +98,13 @@ module "jupyterhub" { shared-pvc = module.jupyterhub-nfs-mount.persistent_volume_claim.name - conda-store-pvc = module.conda-store-nfs-mount.persistent_volume_claim.name - conda-store-mount = "/home/conda" - conda-store-environments = var.conda-store-environments - default-conda-store-namespace = var.conda-store-default-namespace - conda-store-cdsdashboard-token = module.kubernetes-conda-store-server.service-tokens.cdsdashboards - conda-store-service-name = module.kubernetes-conda-store-server.service_name + conda-store-pvc = module.conda-store-nfs-mount.persistent_volume_claim.name + conda-store-mount = "/home/conda" + conda-store-environments = var.conda-store-environments + default-conda-store-namespace = var.conda-store-default-namespace + conda-store-cdsdashboard-token = module.kubernetes-conda-store-server.service-tokens.cdsdashboards + conda-store-argo-workflows-jupyter-scheduler-token = module.kubernetes-conda-store-server.service-tokens.argo-workflows-jupyter-scheduler + conda-store-service-name = module.kubernetes-conda-store-server.service_name extra-mounts = { "/etc/dask" = { diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl index 1d1aac70b4..d975a98fa7 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl @@ -41,3 +41,4 @@ from argo_workflows_executor import ArgoExecutor, ArgoScheduler c.Scheduler.execution_manager_class=ArgoExecutor c.SchedulerApp.scheduler_class=ArgoScheduler +c.SchedulerApp.scheduler_class.use_conda_store_env=True diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index 4c387d3c5f..539e135981 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -345,6 +345,27 @@ def profile_argo_token(groups): } +def profile_conda_store_viewer_token(): + return { + "CONDA_STORE_TOKEN": { + "valueFrom": { + "secretKeyRef": { + "name": "argo-workflows-conda-store-token", + "key": "conda-store-api-token", + } + } + }, + "CONDA_STORE_SERVICE": { + "valueFrom": { + "secretKeyRef": { + "name": "argo-workflows-conda-store-token", + "key": "conda-store-service-name", + } + } + }, + } + + def render_profile(profile, username, groups, keycloak_profilenames): """Render each profile for user. @@ -400,7 +421,12 @@ def render_profile(profile, username, groups, keycloak_profilenames): def preserve_envvars(spawner): # This adds in JUPYTERHUB_ANYONE/GROUP rather than overwrite all env vars, # if set in the spawner for a dashboard to control access. - return {**envvars_fixed, **spawner.environment, **profile_argo_token(groups)} + return { + **envvars_fixed, + **spawner.environment, + **profile_argo_token(groups), + **profile_conda_store_viewer_token(), + } profile["kubespawner_override"]["environment"] = preserve_envvars diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf index 00702a871d..6d7bf78c2b 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf @@ -35,6 +35,7 @@ resource "helm_release" "jupyterhub" { default-conda-store-namespace = var.default-conda-store-namespace conda-store-service-name = var.conda-store-service-name conda-store-cdsdashboards = var.conda-store-cdsdashboard-token + conda-store-jupyter-scheduler = var.conda-store-argo-workflows-jupyter-scheduler-token skel-mount = { name = kubernetes_config_map.etc-skel.metadata.0.name namespace = kubernetes_config_map.etc-skel.metadata.0.namespace @@ -210,3 +211,18 @@ module "jupyterhub-openid-client" { ] jupyterlab_profiles_mapper = true } + + +resource "kubernetes_secret" "argo-workflows-conda-store-token" { + metadata { + name = "argo-workflows-conda-store-token" + namespace = var.namespace + } + + data = { + "conda-store-api-token" = var.conda-store-argo-workflows-jupyter-scheduler-token + "conda-store-service-name" = var.conda-store-service-name + } + + type = "Opaque" +} diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/variables.tf b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/variables.tf index 63ad161ce9..8c0af0b77c 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/variables.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/variables.tf @@ -128,6 +128,11 @@ variable "conda-store-cdsdashboard-token" { default = "" } +variable "conda-store-argo-workflows-jupyter-scheduler-token" { + description = "Token for argo-workflows-jupyter-schedule to use conda-store" + type = string +} + variable "jupyterhub-logout-redirect-url" { description = "Next redirect destination following a Keycloak logout" type = string From 124bdca43445675495b9a716e2c1fc0c8c66f861 Mon Sep 17 00:00:00 2001 From: iameskild Date: Tue, 13 Jun 2023 14:51:55 +0200 Subject: [PATCH 04/12] Update jupyter_server_config --- .../jupyterhub/files/jupyter/jupyter_server_config.py.tpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl index d975a98fa7..0eee557be3 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl @@ -37,7 +37,8 @@ c.NotebookApp.shutdown_no_activity_timeout = ${server_shutdown_no_activity_timeo # JupyterHub idle culler total timeout corresponds (approximately) to: # max(cull_idle_timeout, cull_inactive_timeout) + shutdown_no_activity_timeout -from argo_workflows_executor import ArgoExecutor, ArgoScheduler +from argo_workflows_executor.executor import ArgoExecutor +from argo_workflows_executor.scheduler import ArgoScheduler c.Scheduler.execution_manager_class=ArgoExecutor c.SchedulerApp.scheduler_class=ArgoScheduler From a5e31e823edece2b1a96bc11a3341f1e64e189a8 Mon Sep 17 00:00:00 2001 From: iameskild Date: Tue, 20 Jun 2023 13:17:11 +0200 Subject: [PATCH 05/12] Fix dev, viewer tokens --- .../services/jupyterhub/files/jupyterhub/03-profiles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index 539e135981..a0cdcbed78 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -323,9 +323,9 @@ def profile_argo_token(groups): base = "argo-" if ANALYST in groups: - argo_sa = base + "viewer" + argo_sa = base + "view" if DEVELOPER in groups: - argo_sa = base + "developer" + argo_sa = base + "dev" if ADMIN in groups: argo_sa = base + "admin" else: From 7a15495b235c73e42d761ad1e746027c013f4ed0 Mon Sep 17 00:00:00 2001 From: iameskild Date: Tue, 4 Jul 2023 14:40:20 -0700 Subject: [PATCH 06/12] Update profile_argo_token fn --- .../services/jupyterhub/files/jupyterhub/03-profiles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index a0cdcbed78..b9363e4f69 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -321,6 +321,7 @@ def profile_argo_token(groups): ANALYST = "analyst" base = "argo-" + argo_sa = None if ANALYST in groups: argo_sa = base + "view" @@ -328,7 +329,7 @@ def profile_argo_token(groups): argo_sa = base + "dev" if ADMIN in groups: argo_sa = base + "admin" - else: + if not argo_sa: return {} return { From 5e5aff2ae6d32360d60ffd8a922c8dc6d1244e0f Mon Sep 17 00:00:00 2001 From: iameskild Date: Wed, 12 Jul 2023 12:59:10 -0700 Subject: [PATCH 07/12] Add configmap for valid_argo_roles --- .../services/argo-workflows/main.tf | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf index 565e85c598..34d5819c03 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf @@ -1,6 +1,10 @@ locals { name = "argo-workflows" argo-workflows-prefix = "argo" + # roles + admin = "argo_admin" + dev = "argo_developer" + viewer = "argo_viewer" } resource "helm_release" "argo-workflows" { @@ -83,9 +87,9 @@ module "argo-workflow-openid-client" { client_id = "argo-server-sso" external-url = var.external-url role_mapping = { - "admin" = ["argo_admin"] - "developer" = ["argo_developer"] - "analyst" = ["argo_viewer"] + "admin" = ["${local.admin}"] + "developer" = ["${local.dev}"] + "analyst" = ["${local.viewer}"] } callback-url-paths = [ @@ -186,7 +190,7 @@ resource "kubernetes_service_account_v1" "argo-admin-sa" { name = "argo-admin" namespace = var.namespace annotations = { - "workflows.argoproj.io/rbac-rule" : "'argo_admin' in groups" + "workflows.argoproj.io/rbac-rule" : "'${local.admin}' in groups" "workflows.argoproj.io/rbac-rule-precedence" : "11" } } @@ -226,7 +230,7 @@ resource "kubernetes_service_account_v1" "argo-dev-sa" { name = "argo-dev" namespace = var.namespace annotations = { - "workflows.argoproj.io/rbac-rule" : "'argo_developer' in groups" + "workflows.argoproj.io/rbac-rule" : "'${local.dev}' in groups" "workflows.argoproj.io/rbac-rule-precedence" : "10" } } @@ -266,7 +270,7 @@ resource "kubernetes_service_account_v1" "argo-view-sa" { name = "argo-viewer" namespace = var.namespace annotations = { - "workflows.argoproj.io/rbac-rule" : "'argo_viewer' in groups" + "workflows.argoproj.io/rbac-rule" : "'${local.viewer}' in groups" "workflows.argoproj.io/rbac-rule-precedence" : "9" } } @@ -534,10 +538,25 @@ resource "kubernetes_manifest" "deployment_admission_controller" { "value" = var.namespace }, ] + "volumeMounts" = [ + { + "mountPath" = "/etc/config" + "name" = "valid_argo_roles" + "readOnly" = true + }, + ] "image" = "quay.io/nebari/nebari-workflow-controller:${var.workflow-controller-image-tag}" "name" = "admission-controller" }, ] + "volumes" = [ + { + "name" = "valid_argo_roles" + "configMap" = { + "name" = "valid_argo_roles" + } + }, + ] } } } @@ -566,3 +585,14 @@ resource "kubernetes_manifest" "service_admission_controller" { } } } + +resource "kubernetes_config_map" "valid_argo_roles" { + metadata { + name = "valid_argo_roles" + namespace = var.namespace + } + + data = { + "valid_argo_roles" = jsonencode([local.admin, local.dev]) + } +} From 9cdcd16ea87f4e3c22dc94ad5612e17c44ffe1c2 Mon Sep 17 00:00:00 2001 From: iameskild Date: Wed, 12 Jul 2023 13:56:05 -0700 Subject: [PATCH 08/12] update valid-argo-roles name --- .../kubernetes/services/argo-workflows/main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf index 34d5819c03..216d68d474 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf @@ -541,7 +541,7 @@ resource "kubernetes_manifest" "deployment_admission_controller" { "volumeMounts" = [ { "mountPath" = "/etc/config" - "name" = "valid_argo_roles" + "name" = "valid-argo-roles" "readOnly" = true }, ] @@ -551,9 +551,9 @@ resource "kubernetes_manifest" "deployment_admission_controller" { ] "volumes" = [ { - "name" = "valid_argo_roles" + "name" = "valid-argo-roles" "configMap" = { - "name" = "valid_argo_roles" + "name" = "valid-argo-roles" } }, ] @@ -586,13 +586,13 @@ resource "kubernetes_manifest" "service_admission_controller" { } } -resource "kubernetes_config_map" "valid_argo_roles" { +resource "kubernetes_config_map" "valid-argo-roles" { metadata { - name = "valid_argo_roles" + name = "valid-argo-roles" namespace = var.namespace } data = { - "valid_argo_roles" = jsonencode([local.admin, local.dev]) + "valid-argo-roles" = jsonencode([local.admin, local.dev]) } } From ff864ae799416c0cf085edef01deb5b0e0d5ba34 Mon Sep 17 00:00:00 2001 From: iameskild Date: Thu, 13 Jul 2023 08:39:02 -0700 Subject: [PATCH 09/12] standardize argo naming --- .../services/argo-workflows/main.tf | 36 +++++++++---------- .../files/jupyterhub/03-profiles.py | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf index 216d68d474..91985cb5ac 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf @@ -2,9 +2,9 @@ locals { name = "argo-workflows" argo-workflows-prefix = "argo" # roles - admin = "argo_admin" - dev = "argo_developer" - viewer = "argo_viewer" + admin = "argo-admin" + developer = "argo-developer" + viewer = "argo-viewer" } resource "helm_release" "argo-workflows" { @@ -88,7 +88,7 @@ module "argo-workflow-openid-client" { external-url = var.external-url role_mapping = { "admin" = ["${local.admin}"] - "developer" = ["${local.dev}"] + "developer" = ["${local.developer}"] "analyst" = ["${local.viewer}"] } @@ -187,7 +187,7 @@ resource "kubernetes_manifest" "argo-workflows-ingress-route" { resource "kubernetes_service_account_v1" "argo-admin-sa" { metadata { - name = "argo-admin" + name = local.admin namespace = var.namespace annotations = { "workflows.argoproj.io/rbac-rule" : "'${local.admin}' in groups" @@ -196,9 +196,9 @@ resource "kubernetes_service_account_v1" "argo-admin-sa" { } } -resource "kubernetes_secret_v1" "argo_admin_sa_token" { +resource "kubernetes_secret_v1" "argo-admin-sa-token" { metadata { - name = "argo-admin.service-account-token" + name = "${local.admin}.service-account-token" namespace = var.namespace annotations = { "kubernetes.io/service-account.name" = kubernetes_service_account_v1.argo-admin-sa.metadata[0].name @@ -210,7 +210,7 @@ resource "kubernetes_secret_v1" "argo_admin_sa_token" { resource "kubernetes_cluster_role_binding" "argo-admin-rb" { metadata { - name = "argo-admin" + name = local.admin } role_ref { @@ -225,12 +225,12 @@ resource "kubernetes_cluster_role_binding" "argo-admin-rb" { } } -resource "kubernetes_service_account_v1" "argo-dev-sa" { +resource "kubernetes_service_account_v1" "argo-developer-sa" { metadata { - name = "argo-dev" + name = local.developer namespace = var.namespace annotations = { - "workflows.argoproj.io/rbac-rule" : "'${local.dev}' in groups" + "workflows.argoproj.io/rbac-rule" : "'${local.developer}' in groups" "workflows.argoproj.io/rbac-rule-precedence" : "10" } } @@ -238,18 +238,18 @@ resource "kubernetes_service_account_v1" "argo-dev-sa" { resource "kubernetes_secret_v1" "argo_dev_sa_token" { metadata { - name = "argo-dev.service-account-token" + name = "${local.developer}.service-account-token" namespace = var.namespace annotations = { - "kubernetes.io/service-account.name" = kubernetes_service_account_v1.argo-dev-sa.metadata[0].name + "kubernetes.io/service-account.name" = kubernetes_service_account_v1.argo-developer-sa.metadata[0].name } } type = "kubernetes.io/service-account-token" } -resource "kubernetes_cluster_role_binding" "argo-dev-rb" { +resource "kubernetes_cluster_role_binding" "argo-developer-rb" { metadata { - name = "argo-dev" + name = local.developer } role_ref { @@ -259,7 +259,7 @@ resource "kubernetes_cluster_role_binding" "argo-dev-rb" { } subject { kind = "ServiceAccount" - name = kubernetes_service_account_v1.argo-dev-sa.metadata.0.name + name = kubernetes_service_account_v1.argo-developer-sa.metadata.0.name namespace = var.namespace } } @@ -276,7 +276,7 @@ resource "kubernetes_service_account_v1" "argo-view-sa" { } } -resource "kubernetes_secret_v1" "argo_viewer_sa_token" { +resource "kubernetes_secret_v1" "argo-viewer-sa-token" { metadata { name = "argo-viewer.service-account-token" namespace = var.namespace @@ -593,6 +593,6 @@ resource "kubernetes_config_map" "valid-argo-roles" { } data = { - "valid-argo-roles" = jsonencode([local.admin, local.dev]) + "valid-argo-roles" = jsonencode([local.admin, local.developer]) } } diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index b9363e4f69..3f5f637e9b 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -326,7 +326,7 @@ def profile_argo_token(groups): if ANALYST in groups: argo_sa = base + "view" if DEVELOPER in groups: - argo_sa = base + "dev" + argo_sa = base + "developer" if ADMIN in groups: argo_sa = base + "admin" if not argo_sa: From c21afa413141d8f3e4550649e126a28f37c2306a Mon Sep 17 00:00:00 2001 From: iameskild Date: Tue, 18 Jul 2023 16:44:11 -0700 Subject: [PATCH 10/12] Use /etc/argo instead --- .../modules/kubernetes/services/argo-workflows/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf index 91985cb5ac..29f27da26a 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf @@ -540,7 +540,7 @@ resource "kubernetes_manifest" "deployment_admission_controller" { ] "volumeMounts" = [ { - "mountPath" = "/etc/config" + "mountPath" = "/etc/argo" "name" = "valid-argo-roles" "readOnly" = true }, From 80bf287e616f58df7e513ea52db3cadd33211009 Mon Sep 17 00:00:00 2001 From: iameskild Date: Tue, 18 Jul 2023 18:59:02 -0700 Subject: [PATCH 11/12] Add ARGO_NAMESPACE to env vars --- .../services/jupyterhub/files/jupyterhub/03-profiles.py | 2 ++ .../modules/kubernetes/services/jupyterhub/main.tf | 1 + 2 files changed, 3 insertions(+) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index 3f5f637e9b..46e1d8629e 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -315,6 +315,7 @@ def profile_argo_token(groups): # TODO: create a more robust check user's Argo-Workflow role domain = z2jh.get_config("custom.external-url") + namespace = z2jh.get_config("custom.namespace") ADMIN = "admin" DEVELOPER = "developer" @@ -335,6 +336,7 @@ def profile_argo_token(groups): return { "ARGO_BASE_HREF": "/argo", "ARGO_SERVER": f"{domain}:443", + "ARGO_NAMESPACE": namespace, "ARGO_TOKEN": { "valueFrom": { "secretKeyRef": { diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf index 6d7bf78c2b..ef2bfb9c50 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/main.tf @@ -24,6 +24,7 @@ resource "helm_release" "jupyterhub" { jsonencode({ # custom values can be accessed via z2jh.get_config('custom.') custom = { + namespace = var.namespace external-url = var.external-url theme = var.theme profiles = var.profiles From 87e11f06e99050be0568ce8708de634e576b1cb8 Mon Sep 17 00:00:00 2001 From: iameskild Date: Wed, 19 Jul 2023 22:37:10 -0700 Subject: [PATCH 12/12] Update name to argo_jupyter_scheduler --- .../jupyterhub/files/jupyter/jupyter_server_config.py.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl index 0eee557be3..62270bb602 100644 --- a/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl +++ b/src/_nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyter/jupyter_server_config.py.tpl @@ -37,8 +37,8 @@ c.NotebookApp.shutdown_no_activity_timeout = ${server_shutdown_no_activity_timeo # JupyterHub idle culler total timeout corresponds (approximately) to: # max(cull_idle_timeout, cull_inactive_timeout) + shutdown_no_activity_timeout -from argo_workflows_executor.executor import ArgoExecutor -from argo_workflows_executor.scheduler import ArgoScheduler +from argo_jupyter_scheduler.executor import ArgoExecutor +from argo_jupyter_scheduler.scheduler import ArgoScheduler c.Scheduler.execution_manager_class=ArgoExecutor c.SchedulerApp.scheduler_class=ArgoScheduler