From b9de4a19c0715c81b856124ab658338ae16c6cbd Mon Sep 17 00:00:00 2001 From: Sebastian Widmer Date: Tue, 14 Dec 2021 12:46:44 +0100 Subject: [PATCH] Initial implementation (#3) * Initial implementation * Add alias parameter to the makefile to test the instantiation --- .github/workflows/test.yaml | 2 + .sync.yml | 6 + Makefile | 1 + Makefile.custom.mk | 2 + class/defaults.yml | 37 +- class/vcluster.yml | 2 +- component/app.jsonnet | 5 +- component/cluster.libsonnet | 352 +++++++++++++++ component/main.jsonnet | 5 + component/ocp-route.libsonnet | 120 +++++ component/ocp-route/create-route.sh | 27 ++ .../ROOT/pages/references/parameters.adoc | 158 ++++++- tests/defaults.yml | 6 +- .../apps/defaults.yaml} | 0 .../defaults/defaults/00_namespace.yaml | 7 + .../defaults/defaults/10_cluster.yaml | 224 ++++++++++ tests/golden/oidc/oidc/apps/oidc.yaml | 0 tests/golden/oidc/oidc/oidc/00_namespace.yaml | 7 + tests/golden/oidc/oidc/oidc/10_cluster.yaml | 242 +++++++++++ .../openshift/openshift/apps/openshift.yaml | 0 .../openshift/openshift/00_namespace.yaml | 7 + .../openshift/openshift/10_cluster.yaml | 410 ++++++++++++++++++ tests/oidc.yml | 14 + tests/openshift.yml | 8 + 24 files changed, 1633 insertions(+), 9 deletions(-) create mode 100644 Makefile.custom.mk create mode 100644 component/cluster.libsonnet create mode 100644 component/ocp-route.libsonnet create mode 100644 component/ocp-route/create-route.sh rename tests/golden/defaults/{vcluster/apps/vcluster.yaml => defaults/apps/defaults.yaml} (100%) create mode 100644 tests/golden/defaults/defaults/defaults/00_namespace.yaml create mode 100644 tests/golden/defaults/defaults/defaults/10_cluster.yaml create mode 100644 tests/golden/oidc/oidc/apps/oidc.yaml create mode 100644 tests/golden/oidc/oidc/oidc/00_namespace.yaml create mode 100644 tests/golden/oidc/oidc/oidc/10_cluster.yaml create mode 100644 tests/golden/openshift/openshift/apps/openshift.yaml create mode 100644 tests/golden/openshift/openshift/openshift/00_namespace.yaml create mode 100644 tests/golden/openshift/openshift/openshift/10_cluster.yaml create mode 100644 tests/oidc.yml create mode 100644 tests/openshift.yml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8f2ac8d..00a3c51 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,6 +33,8 @@ jobs: matrix: instance: - defaults + - oidc + - openshift defaults: run: working-directory: ${{ env.COMPONENT_NAME }} diff --git a/.sync.yml b/.sync.yml index 5385417..40d4464 100644 --- a/.sync.yml +++ b/.sync.yml @@ -13,3 +13,9 @@ docs/antora.yml: key: instance entries: - defaults + - oidc + - openshift + +Makefile: + includes: + - Makefile.custom.mk diff --git a/Makefile b/Makefile index c17955d..fde4fbd 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ SHELL := bash .SUFFIXES: include Makefile.vars.mk +include Makefile.custom.mk .PHONY: help help: ## Show this help diff --git a/Makefile.custom.mk b/Makefile.custom.mk new file mode 100644 index 0000000..6e30823 --- /dev/null +++ b/Makefile.custom.mk @@ -0,0 +1,2 @@ +# Configure instance alias for commodore component compile +commodore_args += --alias $(instance) diff --git a/class/defaults.yml b/class/defaults.yml index cd2682d..4b7eefd 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -1,4 +1,39 @@ parameters: vcluster: =_metadata: {} - namespace: syn-vcluster + multi_instance: true + namespace: syn-${_instance} + + host_service_cidr: null + + images: + k3s: + repository: docker.io + image: rancher/k3s + tag: v1.22.2-k3s1 + syncer: + repository: docker.io + image: loftsh/vcluster + tag: 0.5.0-alpha.6 + kubectl: + repository: quay.io + image: bitnami/kubectl + tag: "1.22.4" + + storage: + persistence: true + size: 5Gi + class_name: null + + ingress: + host: null + labels: {} + annotations: {} + + ocp_route: + host: null + + k3s: + additional_args: [] + syncer: + additional_args: [] diff --git a/class/vcluster.yml b/class/vcluster.yml index f8fd4f5..8af4481 100644 --- a/class/vcluster.yml +++ b/class/vcluster.yml @@ -8,4 +8,4 @@ parameters: - input_paths: - vcluster/component/main.jsonnet input_type: jsonnet - output_path: vcluster/ + output_path: ${_instance}/ diff --git a/component/app.jsonnet b/component/app.jsonnet index cb1c33d..c3a8352 100644 --- a/component/app.jsonnet +++ b/component/app.jsonnet @@ -1,10 +1,11 @@ local kap = import 'lib/kapitan.libjsonnet'; local inv = kap.inventory(); local params = inv.parameters.vcluster; +local instance = inv.parameters._instance; local argocd = import 'lib/argocd.libjsonnet'; -local app = argocd.App('vcluster', params.namespace); +local app = argocd.App(instance, params.namespace); { - vcluster: app, + [instance]: app, } diff --git a/component/cluster.libsonnet b/component/cluster.libsonnet new file mode 100644 index 0000000..ad6f549 --- /dev/null +++ b/component/cluster.libsonnet @@ -0,0 +1,352 @@ +local kap = import 'lib/kapitan.libjsonnet'; +local kube = import 'lib/kube.libjsonnet'; +local ocpRoute = import 'ocp-route.libsonnet'; +local inv = kap.inventory(); +// The hiera parameters for the component +local params = inv.parameters.vcluster; + +local isOpenshift = std.startsWith(inv.parameters.facts.distribution, 'openshift'); + +local formatImage = function(img) '%(repository)s/%(image)s:%(tag)s' % img; + +local cluster = function(name, options) + local sa = kube.ServiceAccount('vc-' + name) { + metadata+: { + namespace: options.namespace, + }, + }; + local role = kube.Role(name) { + metadata+: { + namespace: options.namespace, + }, + rules: [ + { + apiGroups: [ + '', + ], + resources: [ + 'configmaps', + 'secrets', + 'services', + 'pods', + 'pods/attach', + 'pods/portforward', + 'pods/exec', + 'endpoints', + 'persistentvolumeclaims', + ], + verbs: [ + 'create', + 'delete', + 'patch', + 'update', + 'get', + 'list', + 'watch', + ], + }, + { + apiGroups: [ + '', + ], + resources: [ + 'events', + 'pods/log', + ], + verbs: [ + 'get', + 'list', + 'watch', + ], + }, + { + apiGroups: [ + 'networking.k8s.io', + ], + resources: [ + 'ingresses', + ], + verbs: [ + 'create', + 'delete', + 'patch', + 'update', + 'get', + 'list', + 'watch', + ], + }, + { + apiGroups: [ + 'apps', + ], + resources: [ + 'statefulsets', + 'replicasets', + 'deployments', + ], + verbs: [ + 'get', + 'list', + 'watch', + ], + }, + ] + if isOpenshift then + [ + { + apiGroups: [ + '', + ], + resources: [ + 'endpoints/restricted', + ], + verbs: [ + 'create', + ], + }, + ] else [], + }; + local roleBinding = kube.RoleBinding(name) { + metadata+: { + namespace: options.namespace, + }, + subjects_: [ sa ], + roleRef_: role, + }; + + local service = kube.Service(name) { + metadata+: { + namespace: options.namespace, + }, + spec: { + type: 'ClusterIP', + ports: [ + { + name: 'https', + port: 443, + targetPort: 8443, + protocol: 'TCP', + }, + ], + selector: { + app: 'vcluster', + release: name, + }, + }, + }; + + local headlessService = kube.Service(name + '-headless') { + metadata+: { + namespace: options.namespace, + }, + spec: { + ports: [ + { + name: 'https', + port: 443, + targetPort: 8443, + protocol: 'TCP', + }, + ], + clusterIP: 'None', + selector: { + app: 'vcluster', + release: name, + }, + }, + }; + + local statefulSet = kube.StatefulSet(name) { + metadata+: { + namespace: options.namespace, + }, + spec: { + serviceName: name + '-headless', + replicas: 1, + selector: { + matchLabels: { + app: 'vcluster', + release: name, + }, + }, + volumeClaimTemplates: + if options.storage.persistence then [ + { + metadata: { + name: 'data', + }, + spec: { + accessModes: [ 'ReadWriteOnce' ], + storageClassName: options.storage.class_name, + resources: { + requests: { + storage: options.storage.size, + }, + }, + }, + }, + ] else [], + template: { + metadata: { + labels: { + app: 'vcluster', + release: name, + }, + }, + spec: { + terminationGracePeriodSeconds: 10, + nodeSelector: {}, + affinity: {}, + tolerations: [], + serviceAccountName: 'vc-' + name, + volumes: if !options.storage.persistence then [ + { + name: 'data', + emptyDir: {}, + }, + ], + containers: [ + { + image: formatImage(options.images.k3s), + name: 'vcluster', + command: [ + '/bin/k3s', + ], + assert options.host_service_cidr != null : ||| + `host_service_cidr` must be set. + + You can find out a host cluster's service CIDR by deploying a service with an invalid ClusterIP (such as 1.1.1.1). + $ kubectl create svc clusterip check-service-cidr --clusterip=1.1.1.1 --tcp=5678:5678 + The error message shows the host cluster's service CIDR: + > The Service "check-service-cidr" is invalid: spec.clusterIPs: Invalid value: []string{"1.1.1.1"}.... The range of valid IPs is 10.96.0.0/12. + |||, + args: [ + 'server', + '--write-kubeconfig=/data/k3s-config/kube-config.yaml', + '--data-dir=/data', + '--disable=traefik,servicelb,metrics-server,local-storage,coredns', + '--disable-network-policy', + '--disable-agent', + '--disable-scheduler', + '--disable-cloud-controller', + '--flannel-backend=none', + '--service-cidr=%s' % options.host_service_cidr, + '--kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle', + ] + options.k3s.additional_args, + env: [], + securityContext: { + allowPrivilegeEscalation: false, + }, + volumeMounts: [ + { + mountPath: '/data', + name: 'data', + }, + ], + resources: { + limits: { + memory: '2Gi', + }, + requests: { + cpu: '200m', + memory: '256Mi', + }, + }, + }, + { + name: 'syncer', + image: formatImage(options.images.syncer), + args: [ + '--name=' + name, + '--out-kube-config-secret=vc-%s-kubeconfig' % name, + ] + options.syncer.additional_args, + livenessProbe: { + httpGet: { + path: '/healthz', + port: 8443, + scheme: 'HTTPS', + }, + failureThreshold: 10, + initialDelaySeconds: 60, + periodSeconds: 2, + }, + readinessProbe: { + httpGet: { + path: '/readyz', + port: 8443, + scheme: 'HTTPS', + }, + failureThreshold: 30, + periodSeconds: 2, + }, + securityContext: { + allowPrivilegeEscalation: false, + }, + env: [], + volumeMounts: [ + { + mountPath: '/data', + name: 'data', + readOnly: true, + }, + ], + resources: { + limits: { + memory: '1Gi', + }, + requests: { + cpu: '100m', + memory: '128Mi', + }, + }, + }, + ], + }, + }, + }, + }; + + local ingress = kube._Object('networking.k8s.io/v1', 'Ingress', name) { + metadata+: { + namespace: options.namespace, + annotations+: options.ingress.annotations, + labels+: options.ingress.labels, + }, + spec: { + rules: [ + { + host: options.ingress.host, + http: { + paths: [ + { + backend: { + service: { + name: name, + port: { + name: 'https', + }, + }, + }, + path: '/', + pathType: 'Prefix', + }, + ], + }, + }, + ], + }, + }; + + std.filter(function(m) m != null, [ + sa, + role, + roleBinding, + service, + headlessService, + statefulSet, + if options.ingress.host != null then ingress, + ] + if options.ocp_route.host != null then ocpRoute.RouteCreateJob(name, 'vc-%s-kubeconfig' % name, options.ocp_route.host, options) else []); + +{ + Cluster: cluster, +} diff --git a/component/main.jsonnet b/component/main.jsonnet index 2342c0c..5c12f7a 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -1,10 +1,15 @@ // main template for vcluster +local cluster = import 'cluster.libsonnet'; +local com = import 'lib/commodore.libjsonnet'; local kap = import 'lib/kapitan.libjsonnet'; local kube = import 'lib/kube.libjsonnet'; local inv = kap.inventory(); // The hiera parameters for the component local params = inv.parameters.vcluster; +local instance = inv.parameters._instance; // Define outputs below { + '00_namespace': kube.Namespace(params.namespace), + '10_cluster': cluster.Cluster(instance, params), } diff --git a/component/ocp-route.libsonnet b/component/ocp-route.libsonnet new file mode 100644 index 0000000..863508e --- /dev/null +++ b/component/ocp-route.libsonnet @@ -0,0 +1,120 @@ +/* +* Creates a re-encrypting OCP route. +* +* Routes do not support reading certificates from secrets. Thus +* certificates have to be known before creating a route. +* The clusters serving certificate is only known after startup. +* So we create a job that: +* - Mounts the vclusters kubeconfig +* - Reads the clusters self signed serving certificate from it +* - Inserts the certificate into the route template +* - Creates the route and sets ownership to the vcluster StatefulSet +*/ + +local kap = import 'lib/kapitan.libjsonnet'; +local kube = import 'lib/kube.libjsonnet'; +local inv = kap.inventory(); +// The hiera parameters for the component +local params = inv.parameters.vcluster; + +local script = importstr './ocp-route/create-route.sh'; + +local routeCreateJob = function(name, secretName, host, options) + local jobName = name + '-create-route'; + + local role = kube.Role(jobName) { + metadata+: { namespace: options.namespace }, + rules: [ + { + apiGroups: [ 'route.openshift.io' ], + resources: [ 'routes', 'routes/custom-host' ], + verbs: [ '*' ], + }, + { + apiGroups: [ 'apps' ], + resources: [ 'statefulsets' ], + resourceNames: [ name ], + verbs: [ 'get' ], + }, + ], + }; + + local serviceAccount = kube.ServiceAccount(jobName) { + metadata+: { namespace: options.namespace }, + }; + + local roleBinding = kube.RoleBinding(jobName) { + metadata+: { namespace: options.namespace }, + subjects_: [ serviceAccount ], + roleRef_: role, + }; + + local routeTemplate = std.manifestJsonEx(kube._Object('route.openshift.io/v1', 'Route', name) { + metadata+: { + namespace: options.namespace, + }, + spec: { + host: host, + path: '/', + port: { + targetPort: 'https', + }, + tls: { + insecureEdgeTerminationPolicy: 'None', + termination: 'reencrypt', + }, + to: { + kind: 'Service', + name: name, + weight: 100, + }, + wildcardPolicy: 'None', + }, + }, ''); + + local job = kube.Job(jobName) { + metadata+: { + namespace: options.namespace, + annotations+: { + 'argocd.argoproj.io/hook': 'PostSync', + }, + }, + spec+: { + template+: { + spec+: { + serviceAccountName: serviceAccount.metadata.name, + containers_+: { + patch_crds: kube.Container(name) { + image: '%s/%s:%s' % [ options.images.kubectl.repository, options.images.kubectl.image, options.images.kubectl.tag ], + workingDir: '/export', + command: [ 'sh' ], + args: [ '-eu', '-c', script, '--', routeTemplate ], + env: [ + { name: 'HOME', value: '/export' }, + { name: 'NAMESPACE', value: options.namespace }, + { name: 'VCLUSTER_STS_NAME', value: name }, + ], + volumeMounts: [ + { name: 'export', mountPath: '/export' }, + { name: 'kubeconfig', mountPath: '/etc/vcluster-kubeconfig', readOnly: true }, + ], + }, + }, + volumes+: [ + { name: 'export', emptyDir: {} }, + { name: 'kubeconfig', secret: { secretName: secretName } }, + ], + }, + }, + }, + }; + [ + serviceAccount, + role, + roleBinding, + job, + ]; + +{ + RouteCreateJob: routeCreateJob, +} diff --git a/component/ocp-route/create-route.sh b/component/ocp-route/create-route.sh new file mode 100644 index 0000000..62a289f --- /dev/null +++ b/component/ocp-route/create-route.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu + +vcluster_kubeconfig=/etc/vcluster-kubeconfig/config + +echo "Using kubeconfig: $vcluster_kubeconfig" + +cert=$(kubectl --kubeconfig $vcluster_kubeconfig config view '-o=template={{(index (index .clusters 0).cluster "certificate-authority-data") | base64decode}}' --raw) + +echo "Found certificate:\n$cert" + +echo "Looking for StatefulSet.apps/${VCLUSTER_STS_NAME}..." + +owner=$(kubectl get StatefulSet.apps "$VCLUSTER_STS_NAME" -ojson | jq '{kind: .kind, apiVersion: .apiVersion, name: .metadata.name, uid: .metadata.uid}') + +echo "Found StatefulSet as owner: $owner" + +echo "Applying route..." + +printf "$1" \ + | jq \ + --arg cert "$cert" \ + --argjson owner "$owner" \ + '.metadata.ownerReferences = [$owner] | .spec.tls.destinationCACertificate = $cert' \ + | kubectl apply -f - -oyaml + +echo "Done!" diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index 2e09c79..e9ce5d7 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -2,18 +2,172 @@ The parent key for all of the following parameters is `vcluster`. +This component supports multi-instantiation. + == `namespace` [horizontal] type:: string -default:: `syn-vcluster` +default:: `syn-${_instance}` The namespace in which to deploy this component. +Deploying multiple vclusters in the same namespace isn't supported. + +== `host_service_cidr` + +[horizontal] +type:: string +default:: `null` + +The host cluster's service CIDR. Must be set. + +You can find out a host cluster's service CIDR by deploying a service with an invalid ClusterIP (such as 1.1.1.1). + +[source,shell] +---- +kubectl create svc clusterip check-service-cidr --clusterip=1.1.1.1 --tcp=5678:5678 +---- + +The error message shows the host cluster's service CIDR: + +[source] +---- +The Service "check-service-cidr" is invalid: spec.clusterIPs: Invalid value: []string{"1.1.1.1"}.... The range of valid IPs is 10.96.0.0/12. +---- + + +== `images` + +[horizontal] +type:: dictionary + +Dictionary containing the container images used by this component. + +The `kubectl` image is used to create OCP routes. The `kubectl` and `jq` binaries are required in this image. + + +== `k3s.additional_args` + +[horizontal] +type:: list +default:: `[]` +example:: ++ +[source,yaml] +---- +k3s: + additional_args: + - --kube-apiserver-arg=oidc-issuer-url=https://id.local/auth/realms/local + - --kube-apiserver-arg=oidc-client-id=local + - --kube-apiserver-arg=oidc-username-claim=email + - --kube-apiserver-arg=oidc-groups-claim=groups +---- + +Additional arguments for the k3s cluster. + + +== `syncer.additional_args` + +[horizontal] +type:: list +default:: `[]` + +Additional arguments for vcluster syncer. + + +== `storage.persistence` + +[horizontal] +type:: boolean +default:: `true` + +Persistence controls whether vcluster resources are persisted between deployments. + + +== `storage.size` + +[horizontal] +type:: string +default:: `5Gi` + +The size of the persistent volume claim. + + +== `storage.class_name` + +[horizontal] +type:: string +default:: `null` + +The `StorageClass` used for the persistent volume claim. + + +== `ingress.host` + +[horizontal] +type:: string +default:: `null` + +If set, an ingress with the defined host is created. + + +== `ingress.annotations` + +[horizontal] +type:: dict +default:: `{}` +example:: ++ +[source,yaml] +---- +ingress: + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/backend-protocol: HTTPS + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "true" +---- + +Additional annotations for the ingress object. + + +== `ingress.labels` + +[horizontal] +type:: dict +default:: `{}` + +Additional labels for the ingress object. + + +== `ocp_route.host` + +[horizontal] +type:: string +default:: `null` + +If set, a `route.openshift.io/v1.Route` with the defined host is created. + +The route is set with TLS termination set to re-encrypt. + +The re-encyption breaks the mTLS user authentication. +A secondary authentication method, like OIDC, should be configured. + == Example [source,yaml] ---- -namespace: example-namespace +host_service_cidr: 172.30.0.0/16 +storage: + persistence: false +ingress: + host: testcluster.local +k3s: + additional_args: + - --kube-apiserver-arg=oidc-issuer-url=https://id.local/auth/realms/local + - --kube-apiserver-arg=oidc-client-id=local + - --kube-apiserver-arg=oidc-username-claim=email + - --kube-apiserver-arg=oidc-groups-claim=groups ---- diff --git a/tests/defaults.yml b/tests/defaults.yml index a4da5b7..ad3d80a 100644 --- a/tests/defaults.yml +++ b/tests/defaults.yml @@ -1,3 +1,3 @@ -# Overwrite parameters here - -# parameters: {...} +parameters: + vcluster: + host_service_cidr: 172.30.0.0/16 diff --git a/tests/golden/defaults/vcluster/apps/vcluster.yaml b/tests/golden/defaults/defaults/apps/defaults.yaml similarity index 100% rename from tests/golden/defaults/vcluster/apps/vcluster.yaml rename to tests/golden/defaults/defaults/apps/defaults.yaml diff --git a/tests/golden/defaults/defaults/defaults/00_namespace.yaml b/tests/golden/defaults/defaults/defaults/00_namespace.yaml new file mode 100644 index 0000000..3346cb5 --- /dev/null +++ b/tests/golden/defaults/defaults/defaults/00_namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: {} + labels: + name: syn-defaults + name: syn-defaults diff --git a/tests/golden/defaults/defaults/defaults/10_cluster.yaml b/tests/golden/defaults/defaults/defaults/10_cluster.yaml new file mode 100644 index 0000000..3c4e76f --- /dev/null +++ b/tests/golden/defaults/defaults/defaults/10_cluster.yaml @@ -0,0 +1,224 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + name: vc-defaults + name: vc-defaults + namespace: syn-defaults +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + name: defaults + name: defaults + namespace: syn-defaults +rules: + - apiGroups: + - '' + resources: + - configmaps + - secrets + - services + - pods + - pods/attach + - pods/portforward + - pods/exec + - endpoints + - persistentvolumeclaims + verbs: + - create + - delete + - patch + - update + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + - pods/log + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - patch + - update + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - replicasets + - deployments + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + name: defaults + name: defaults + namespace: syn-defaults +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: defaults +subjects: + - kind: ServiceAccount + name: vc-defaults + namespace: syn-defaults +--- +apiVersion: v1 +kind: Service +metadata: + annotations: {} + labels: + name: defaults + name: defaults + namespace: syn-defaults +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: vcluster + release: defaults + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + annotations: {} + labels: + name: defaults-headless + name: defaults-headless + namespace: syn-defaults +spec: + clusterIP: None + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: vcluster + release: defaults +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: {} + labels: + name: defaults + name: defaults + namespace: syn-defaults +spec: + replicas: 1 + selector: + matchLabels: + app: vcluster + release: defaults + serviceName: defaults-headless + template: + metadata: + labels: + app: vcluster + release: defaults + spec: + affinity: {} + containers: + - args: + - server + - --write-kubeconfig=/data/k3s-config/kube-config.yaml + - --data-dir=/data + - --disable=traefik,servicelb,metrics-server,local-storage,coredns + - --disable-network-policy + - --disable-agent + - --disable-scheduler + - --disable-cloud-controller + - --flannel-backend=none + - --service-cidr=172.30.0.0/16 + - --kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle + command: + - /bin/k3s + env: [] + image: docker.io/rancher/k3s:v1.22.2-k3s1 + name: vcluster + resources: + limits: + memory: 2Gi + requests: + cpu: 200m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /data + name: data + - args: + - --name=defaults + - --out-kube-config-secret=vc-defaults-kubeconfig + env: [] + image: docker.io/loftsh/vcluster:0.5.0-alpha.6 + livenessProbe: + failureThreshold: 10 + httpGet: + path: /healthz + port: 8443 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 2 + name: syncer + readinessProbe: + failureThreshold: 30 + httpGet: + path: /readyz + port: 8443 + scheme: HTTPS + periodSeconds: 2 + resources: + limits: + memory: 1Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /data + name: data + readOnly: true + nodeSelector: {} + serviceAccountName: vc-defaults + terminationGracePeriodSeconds: 10 + tolerations: [] + volumes: null + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: null diff --git a/tests/golden/oidc/oidc/apps/oidc.yaml b/tests/golden/oidc/oidc/apps/oidc.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/oidc/oidc/oidc/00_namespace.yaml b/tests/golden/oidc/oidc/oidc/00_namespace.yaml new file mode 100644 index 0000000..6b1321a --- /dev/null +++ b/tests/golden/oidc/oidc/oidc/00_namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: {} + labels: + name: testns + name: testns diff --git a/tests/golden/oidc/oidc/oidc/10_cluster.yaml b/tests/golden/oidc/oidc/oidc/10_cluster.yaml new file mode 100644 index 0000000..354077f --- /dev/null +++ b/tests/golden/oidc/oidc/oidc/10_cluster.yaml @@ -0,0 +1,242 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + name: vc-oidc + name: vc-oidc + namespace: testns +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + name: oidc + name: oidc + namespace: testns +rules: + - apiGroups: + - '' + resources: + - configmaps + - secrets + - services + - pods + - pods/attach + - pods/portforward + - pods/exec + - endpoints + - persistentvolumeclaims + verbs: + - create + - delete + - patch + - update + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + - pods/log + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - patch + - update + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - replicasets + - deployments + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + name: oidc + name: oidc + namespace: testns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: oidc +subjects: + - kind: ServiceAccount + name: vc-oidc + namespace: testns +--- +apiVersion: v1 +kind: Service +metadata: + annotations: {} + labels: + name: oidc + name: oidc + namespace: testns +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: vcluster + release: oidc + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + annotations: {} + labels: + name: oidc-headless + name: oidc-headless + namespace: testns +spec: + clusterIP: None + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: vcluster + release: oidc +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: {} + labels: + name: oidc + name: oidc + namespace: testns +spec: + replicas: 1 + selector: + matchLabels: + app: vcluster + release: oidc + serviceName: oidc-headless + template: + metadata: + labels: + app: vcluster + release: oidc + spec: + affinity: {} + containers: + - args: + - server + - --write-kubeconfig=/data/k3s-config/kube-config.yaml + - --data-dir=/data + - --disable=traefik,servicelb,metrics-server,local-storage,coredns + - --disable-network-policy + - --disable-agent + - --disable-scheduler + - --disable-cloud-controller + - --flannel-backend=none + - --service-cidr=172.30.0.0/16 + - --kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle + - --kube-apiserver-arg=oidc-issuer-url=https://id.local/auth/realms/local + - --kube-apiserver-arg=oidc-client-id=local + - --kube-apiserver-arg=oidc-username-claim=email + - --kube-apiserver-arg=oidc-groups-claim=groups + command: + - /bin/k3s + env: [] + image: docker.io/rancher/k3s:v1.22.2-k3s1 + name: vcluster + resources: + limits: + memory: 2Gi + requests: + cpu: 200m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /data + name: data + - args: + - --name=oidc + - --out-kube-config-secret=vc-oidc-kubeconfig + env: [] + image: docker.io/loftsh/vcluster:0.5.0-alpha.6 + livenessProbe: + failureThreshold: 10 + httpGet: + path: /healthz + port: 8443 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 2 + name: syncer + readinessProbe: + failureThreshold: 30 + httpGet: + path: /readyz + port: 8443 + scheme: HTTPS + periodSeconds: 2 + resources: + limits: + memory: 1Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /data + name: data + readOnly: true + nodeSelector: {} + serviceAccountName: vc-oidc + terminationGracePeriodSeconds: 10 + tolerations: [] + volumes: + - emptyDir: {} + name: data + volumeClaimTemplates: [] +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: {} + labels: + name: oidc + name: oidc + namespace: testns +spec: + rules: + - host: testcluster.local + http: + paths: + - backend: + service: + name: oidc + port: + name: https + path: / + pathType: Prefix diff --git a/tests/golden/openshift/openshift/apps/openshift.yaml b/tests/golden/openshift/openshift/apps/openshift.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/openshift/openshift/openshift/00_namespace.yaml b/tests/golden/openshift/openshift/openshift/00_namespace.yaml new file mode 100644 index 0000000..e6958f6 --- /dev/null +++ b/tests/golden/openshift/openshift/openshift/00_namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: {} + labels: + name: syn-openshift + name: syn-openshift diff --git a/tests/golden/openshift/openshift/openshift/10_cluster.yaml b/tests/golden/openshift/openshift/openshift/10_cluster.yaml new file mode 100644 index 0000000..a6e6efa --- /dev/null +++ b/tests/golden/openshift/openshift/openshift/10_cluster.yaml @@ -0,0 +1,410 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + name: vc-openshift + name: vc-openshift + namespace: syn-openshift +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + name: openshift + name: openshift + namespace: syn-openshift +rules: + - apiGroups: + - '' + resources: + - configmaps + - secrets + - services + - pods + - pods/attach + - pods/portforward + - pods/exec + - endpoints + - persistentvolumeclaims + verbs: + - create + - delete + - patch + - update + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + - pods/log + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - patch + - update + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - replicasets + - deployments + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - endpoints/restricted + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + name: openshift + name: openshift + namespace: syn-openshift +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: openshift +subjects: + - kind: ServiceAccount + name: vc-openshift + namespace: syn-openshift +--- +apiVersion: v1 +kind: Service +metadata: + annotations: {} + labels: + name: openshift + name: openshift + namespace: syn-openshift +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: vcluster + release: openshift + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + annotations: {} + labels: + name: openshift-headless + name: openshift-headless + namespace: syn-openshift +spec: + clusterIP: None + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: vcluster + release: openshift +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: {} + labels: + name: openshift + name: openshift + namespace: syn-openshift +spec: + replicas: 1 + selector: + matchLabels: + app: vcluster + release: openshift + serviceName: openshift-headless + template: + metadata: + labels: + app: vcluster + release: openshift + spec: + affinity: {} + containers: + - args: + - server + - --write-kubeconfig=/data/k3s-config/kube-config.yaml + - --data-dir=/data + - --disable=traefik,servicelb,metrics-server,local-storage,coredns + - --disable-network-policy + - --disable-agent + - --disable-scheduler + - --disable-cloud-controller + - --flannel-backend=none + - --service-cidr=172.30.0.0/16 + - --kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle + command: + - /bin/k3s + env: [] + image: docker.io/rancher/k3s:v1.22.2-k3s1 + name: vcluster + resources: + limits: + memory: 2Gi + requests: + cpu: 200m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /data + name: data + - args: + - --name=openshift + - --out-kube-config-secret=vc-openshift-kubeconfig + env: [] + image: docker.io/loftsh/vcluster:0.5.0-alpha.6 + livenessProbe: + failureThreshold: 10 + httpGet: + path: /healthz + port: 8443 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 2 + name: syncer + readinessProbe: + failureThreshold: 30 + httpGet: + path: /readyz + port: 8443 + scheme: HTTPS + periodSeconds: 2 + resources: + limits: + memory: 1Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /data + name: data + readOnly: true + nodeSelector: {} + serviceAccountName: vc-openshift + terminationGracePeriodSeconds: 10 + tolerations: [] + volumes: null + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: null +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + name: openshift-create-route + name: openshift-create-route + namespace: syn-openshift +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + name: openshift-create-route + name: openshift-create-route + namespace: syn-openshift +rules: + - apiGroups: + - route.openshift.io + resources: + - routes + - routes/custom-host + verbs: + - '*' + - apiGroups: + - apps + resourceNames: + - openshift + resources: + - statefulsets + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + name: openshift-create-route + name: openshift-create-route + namespace: syn-openshift +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: openshift-create-route +subjects: + - kind: ServiceAccount + name: openshift-create-route + namespace: syn-openshift +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + argocd.argoproj.io/hook: PostSync + labels: + name: openshift-create-route + name: openshift-create-route + namespace: syn-openshift +spec: + completions: 1 + parallelism: 1 + template: + metadata: + labels: + name: openshift-create-route + spec: + containers: + - args: + - -eu + - -c + - "#!/bin/sh\nset -eu\n\nvcluster_kubeconfig=/etc/vcluster-kubeconfig/config\n\ + \necho \"Using kubeconfig: $vcluster_kubeconfig\"\n\ncert=$(kubectl\ + \ --kubeconfig $vcluster_kubeconfig config view '-o=template={{(index\ + \ (index .clusters 0).cluster \"certificate-authority-data\") | base64decode}}'\ + \ --raw)\n\necho \"Found certificate:\\n$cert\"\n\necho \"Looking for\ + \ StatefulSet.apps/${VCLUSTER_STS_NAME}...\"\n\nowner=$(kubectl get\ + \ StatefulSet.apps \"$VCLUSTER_STS_NAME\" -ojson | jq '{kind: .kind,\ + \ apiVersion: .apiVersion, name: .metadata.name, uid: .metadata.uid}')\n\ + \necho \"Found StatefulSet as owner: $owner\"\n\necho \"Applying route...\"\ + \n\nprintf \"$1\" \\\n | jq \\\n --arg cert \"$cert\"\ + \ \\\n --argjson owner \"$owner\" \\\n '.metadata.ownerReferences\ + \ = [$owner] | .spec.tls.destinationCACertificate = $cert' \\\n |\ + \ kubectl apply -f - -oyaml\n\necho \"Done!\"\n" + - -- + - '{ + + "apiVersion": "route.openshift.io/v1", + + "kind": "Route", + + "metadata": { + + "annotations": { + + + }, + + "labels": { + + "name": "openshift" + + }, + + "name": "openshift", + + "namespace": "syn-openshift" + + }, + + "spec": { + + "host": "test.apps.local", + + "path": "/", + + "port": { + + "targetPort": "https" + + }, + + "tls": { + + "insecureEdgeTerminationPolicy": "None", + + "termination": "reencrypt" + + }, + + "to": { + + "kind": "Service", + + "name": "openshift", + + "weight": 100 + + }, + + "wildcardPolicy": "None" + + } + + }' + command: + - sh + env: + - name: HOME + value: /export + - name: NAMESPACE + value: syn-openshift + - name: VCLUSTER_STS_NAME + value: openshift + image: quay.io/bitnami/kubectl:1.22.4 + imagePullPolicy: IfNotPresent + name: openshift + ports: [] + stdin: false + tty: false + volumeMounts: + - mountPath: /export + name: export + - mountPath: /etc/vcluster-kubeconfig + name: kubeconfig + readOnly: true + workingDir: /export + imagePullSecrets: [] + initContainers: [] + restartPolicy: OnFailure + serviceAccountName: openshift-create-route + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: export + - name: kubeconfig + secret: + secretName: vc-openshift-kubeconfig diff --git a/tests/oidc.yml b/tests/oidc.yml new file mode 100644 index 0000000..19d8749 --- /dev/null +++ b/tests/oidc.yml @@ -0,0 +1,14 @@ +parameters: + vcluster: + host_service_cidr: 172.30.0.0/16 + namespace: testns + storage: + persistence: false + ingress: + host: testcluster.local + k3s: + additional_args: + - --kube-apiserver-arg=oidc-issuer-url=https://id.local/auth/realms/local + - --kube-apiserver-arg=oidc-client-id=local + - --kube-apiserver-arg=oidc-username-claim=email + - --kube-apiserver-arg=oidc-groups-claim=groups diff --git a/tests/openshift.yml b/tests/openshift.yml new file mode 100644 index 0000000..31578dc --- /dev/null +++ b/tests/openshift.yml @@ -0,0 +1,8 @@ +parameters: + vcluster: + host_service_cidr: 172.30.0.0/16 + ocp_route: + host: test.apps.local + + facts: + distribution: openshift4