Secrets, scopes, ACL support (#50)

* update to kube 0.92 and openapi 0.22 and related client changes * add script for processing an openapi spec that does not exist * add generated secrets SDK * derive jsonschema for generated sdk * move everything to the same version of the http lib make error glue into a macro to avoid copypaste acrobatix * add databrickssecretscope crd, works on qa tenant * fix gitcredential pagination bug, add initial_manage_principal to scope params * add one-off secrets controller that solely writes * fix/fmt * uncommitted CRD, bump version, update readme * update helm chart
mach-kernel · Jul 16, 2024 · 6833a0b · 6833a0b
1 parent af29e3e
commit 6833a0b
Show file tree

Hide file tree

Showing 72 changed files with 3,706 additions and 580 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -3,5 +3,6 @@ members = [
     "databricks-kube",
     "databricks-rust-jobs",
     "databricks-rust-git-credentials",
-    "databricks-rust-repos"
+    "databricks-rust-repos",
+    "databricks-rust-secrets"
 ]
diff --git a/README.md b/README.md
@@ -10,11 +10,12 @@ coverY: 0
 
 A [kube-rs](https://kube.rs/) operator to enable GitOps style management of Databricks resources. It supports the following APIs:
 
-| API                 | CRD           |
-| ------------------- | ------------- |
-| Jobs 2.1            | DatabricksJob |
-| Git Credentials 2.0 | GitCredential |
-| Repos 2.0           | Repo          |
+| API                 | CRD                                     |
+| ------------------- | --------------------------------------- |
+| Jobs 2.1            | DatabricksJob                           |
+| Git Credentials 2.0 | GitCredential                           |
+| Repos 2.0           | Repo                                    |
+| Secrets 2.0         | DatabricksSecretScope, DatabricksSecret |
 
 Experimental headed towards stable. See the GitHub project board for the roadmap. Contributions and feedback are welcome!
 
@@ -172,39 +173,48 @@ cargo run
 
 The client is generated by `openapi-generator` and then lightly postprocessed so we get models that derive [`JsonSchema`](https://github.com/GREsau/schemars#basic-usage) and fix some bugs.
 
-TODO: Fork or fix generator/template issues instead of sed.
+<details>
+  <summary> TODO: Manual client 'fixes' </summary>
 
-```bash
-# Hey!! This uses GNU sed
-# brew install gnu-sed
+  ```bash
+  # Hey!! This uses GNU sed
+  # brew install gnu-sed
 
-# Jobs API
-openapi-generator generate -g rust -i openapi/jobs-2.1-aws.yaml -c openapi/config-jobs.yaml -o dbr_jobs
+  # Jobs API
+  openapi-generator generate -g rust -i openapi/jobs-2.1-aws.yaml -c openapi/config-jobs.yaml -o dbr_jobs
 
-# Derive JsonSchema for all models and add schemars as dep
-gsed -i -e 's/derive(Clone/derive(JsonSchema, Clone/' dbr_jobs/src/models/*
-gsed -i -e 's/\/\*/use schemars::JsonSchema;\n\/\*/' dbr_jobs/src/models/*
-gsed -r -i -e 's/(\[dependencies\])/\1\nschemars = "0.8.11"/' dbr_jobs/Cargo.toml
+  # Derive JsonSchema for all models and add schemars as dep
+  gsed -i -e 's/derive(Clone/derive(JsonSchema, Clone/' dbr_jobs/src/models/*
+  gsed -i -e 's/\/\*/use schemars::JsonSchema;\n\/\*/' dbr_jobs/src/models/*
+  gsed -r -i -e 's/(\[dependencies\])/\1\nschemars = "0.8.11"/' dbr_jobs/Cargo.toml
 
-# Missing import?
-gsed -r -i -e 's/(use reqwest;)/\1\nuse crate::models::ViewsToExport;/' dbr_jobs/src/apis/default_api.rs
+  # Missing import?
+  gsed -r -i -e 's/(use reqwest;)/\1\nuse crate::models::ViewsToExport;/' dbr_jobs/src/apis/default_api.rs
 
-# Git Credentials API
-openapi-generator generate -g rust -i openapi/gitcredentials-2.0-aws.yaml -c openapi/config-git.yaml -o dbr_git_creds
+  # Git Credentials API
+  openapi-generator generate -g rust -i openapi/gitcredentials-2.0-aws.yaml -c openapi/config-git.yaml -o dbr_git_creds
 
-# Derive JsonSchema for all models and add schemars as dep
-gsed -i -e 's/derive(Clone/derive(JsonSchema, Clone/' dbr_git_creds/src/models/*
-gsed -i -e 's/\/\*/use schemars::JsonSchema;\n\/\*/' dbr_git_creds/src/models/*
-gsed -r -i -e 's/(\[dependencies\])/\1\nschemars = "0.8.11"/' dbr_git_creds/Cargo.toml
+  # Derive JsonSchema for all models and add schemars as dep
+  gsed -i -e 's/derive(Clone/derive(JsonSchema, Clone/' dbr_git_creds/src/models/*
+  gsed -i -e 's/\/\*/use schemars::JsonSchema;\n\/\*/' dbr_git_creds/src/models/*
+  gsed -r -i -e 's/(\[dependencies\])/\1\nschemars = "0.8.11"/' dbr_git_creds/Cargo.toml
 
-# Repos API
-openapi-generator generate -g rust -i openapi/repos-2.0-aws.yaml -c openapi/config-repos.yaml -o dbr_repo
+  # Repos API
+  openapi-generator generate -g rust -i openapi/repos-2.0-aws.yaml -c openapi/config-repos.yaml -o dbr_repo
+
+  # Derive JsonSchema for all models and add schemars as dep
+  gsed -i -e 's/derive(Clone/derive(JsonSchema, Clone/' dbr_repo/src/models/*
+  gsed -i -e 's/\/\*/use schemars::JsonSchema;\n\/\*/' dbr_repo/src/models/*
+  gsed -r -i -e 's/(\[dependencies\])/\1\nschemars = "0.8.11"/' dbr_repo/Cargo.toml
+
+  # Secrets API
+  openapi-generator generate -g rust -i openapi/secrets-aws.yaml -c openapi/config-secrets.yaml -o dbr_secrets
+  sed -i -e 's/derive(Clone/derive(JsonSchema, Clone/' dbr_secrets/src/models/*
+  sed -i -e 's/\/\*/use schemars::JsonSchema;\n\/\*/' dbr_secrets/src/models/*
+  sed -r -i -e 's/(\[dependencies\])/\1\nschemars = "0.8.11"/' dbr_secrets/Cargo.toml
+  ```
+</details>
 
-# Derive JsonSchema for all models and add schemars as dep
-gsed -i -e 's/derive(Clone/derive(JsonSchema, Clone/' dbr_repo/src/models/*
-gsed -i -e 's/\/\*/use schemars::JsonSchema;\n\/\*/' dbr_repo/src/models/*
-gsed -r -i -e 's/(\[dependencies\])/\1\nschemars = "0.8.11"/' dbr_repo/Cargo.toml
-```
 
 ### Expand CRD macros
 

diff --git a/charts/databricks-kube-operator/Chart.yaml b/charts/databricks-kube-operator/Chart.yaml
@@ -1,8 +1,8 @@
 apiVersion: v2
-appVersion: 0.7.0
+appVersion: 0.8.0
 name: databricks-kube-operator
 description: A kube-rs operator for managing Databricks API resources
-version: 0.8.2
+version: 0.9.0
 
 home: https://github.com/mach-kernel/databricks-kube-operator
 sources:

diff --git a/charts/databricks-kube-operator/templates/crds.yaml b/charts/databricks-kube-operator/templates/crds.yaml
@@ -1314,5 +1314,107 @@ spec:
     storage: true
     subresources: {}
 
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: databrickssecretscopes.com.dstancu.databricks
+spec:
+  group: com.dstancu.databricks
+  names:
+    categories: []
+    kind: DatabricksSecretScope
+    plural: databrickssecretscopes
+    shortNames: []
+    singular: databrickssecretscope
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns: []
+    name: v1
+    schema:
+      openAPIV3Schema:
+        description: Auto-generated derived type for DatabricksSecretScopeSpec via `CustomResource`
+        properties:
+          spec:
+            properties:
+              initial_manage_principal:
+                nullable: true
+                type: string
+              scope:
+                properties:
+                  backend_type:
+                    description: The type of secret scope backend.
+                    enum:
+                    - DATABRICKS
+                    - AZURE_KEYVAULT
+                    nullable: true
+                    type: string
+                  keyvault_metadata:
+                    description: The metadata for the secret scope if the type is `AZURE_KEYVAULT`
+                    nullable: true
+                    properties:
+                      dns_name:
+                        description: The DNS of the KeyVault
+                        type: string
+                      resource_id:
+                        description: The resource id of the azure KeyVault that user wants to associate the scope with.
+                        type: string
+                    required:
+                    - dns_name
+                    - resource_id
+                    type: object
+                  name:
+                    description: A unique name to identify the secret scope.
+                    nullable: true
+                    type: string
+                type: object
+            required:
+            - scope
+            type: object
+        required:
+        - spec
+        title: DatabricksSecretScope
+        type: object
+    served: true
+    storage: true
+    subresources: {}
 
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: databrickssecrets.com.dstancu.databricks
+spec:
+  group: com.dstancu.databricks
+  names:
+    categories: []
+    kind: DatabricksSecret
+    plural: databrickssecrets
+    shortNames: []
+    singular: databrickssecret
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns: []
+    name: v1
+    schema:
+      openAPIV3Schema:
+        description: Auto-generated derived type for DatabricksSecretSpec via `CustomResource`
+        properties:
+          spec:
+            properties:
+              scope:
+                type: string
+              secret_name:
+                type: string
+            required:
+            - scope
+            - secret_name
+            type: object
+        required:
+        - spec
+        title: DatabricksSecret
+        type: object
+    served: true
+    storage: true
+    subresources: {}
 {{- end -}}
diff --git a/databricks-kube/Cargo.toml b/databricks-kube/Cargo.toml
@@ -5,13 +5,14 @@ path = "src/crdgen.rs"
 [package]
 name = "databricks_kube"
 default-run = "databricks_kube"
-version = "0.7.0"
+version = "0.8.0"
 edition = "2021"
 
 [dependencies]
 databricks_rust_jobs = { path = "../databricks-rust-jobs" }
 databricks_rust_git_credentials = { path = "../databricks-rust-git-credentials" }
 databricks_rust_repos = { path = "../databricks-rust-repos" }
+databricks_rust_secrets = { path = "../databricks-rust-secrets" }
 
 async-stream = "0.3.3"
 assert-json-diff = "2.0.2"
@@ -20,8 +21,8 @@ env_logger = { version = "0.9.1" }
 futures = "0.3"
 git-version = "0.3.5"
 jsonschema = "0.16.1"
-kube = { version = "0.82.2", features = ["runtime", "derive", "client"] }
-k8s-openapi = { version = "0.18.0", features = ["v1_25"] }
+kube = { version = "0.92.0", features = ["runtime", "derive", "client"] }
+k8s-openapi = { version = "0.22.0", features = ["v1_25"] }
 lazy_static = "1.4.0"
 log = "0.4.17"
 serde = { version = "1.0.147", features = ["derive"] }
@@ -34,9 +35,12 @@ tokio-stream = "0.1.11"
 thiserror = "1.0.56"
 
 [dependencies.reqwest]
-version = "^0.11"
+version = "^0.12"
 features = ["json", "multipart"]
 
 [dev-dependencies]
 tower-test = "0.4.0"
-hyper = "0.14.23"
+hyper = "0.14.23"
+http = "1.1.0"
+http-body = "1.0.0"
+http-body-util = "0.1.2"
diff --git a/databricks-kube/src/crdgen.rs b/databricks-kube/src/crdgen.rs
@@ -20,4 +20,12 @@ fn main() {
         "---\n{}\n",
         to_string(&crate::crds::repo::Repo::crd()).unwrap()
     );
+    print!(
+        "---\n{}\n",
+        to_string(&crate::crds::databricks_secret_scope::DatabricksSecretScope::crd()).unwrap()
+    );
+    print!(
+        "---\n{}\n",
+        to_string(&crate::crds::databricks_secret::DatabricksSecret::crd()).unwrap()
+    );
 }