From ba4e73f45546ecc9b3c3ae7b6fc31a8f132cb647 Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Tue, 6 Dec 2016 00:44:55 +0000 Subject: [PATCH 1/5] Make Kubernetes pods configurable --- kubernetes/catalog/kubernetes/kubernetes.bom | 26 +++++++++----------- kubernetes/catalog/kubernetes/pods.bom | 24 ++++++++++++++++++ 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/kubernetes/catalog/kubernetes/kubernetes.bom b/kubernetes/catalog/kubernetes/kubernetes.bom index afa36dfa..9c7fa489 100644 --- a/kubernetes/catalog/kubernetes/kubernetes.bom +++ b/kubernetes/catalog/kubernetes/kubernetes.bom @@ -240,6 +240,14 @@ brooklyn.catalog: Value is duration in milliseconds. type: long default: 900000 + - name: kubernetes.pods.spec + label: Kubernetes Pods Spec + description: | + An EntitySpec for the set of Kubernetes Pods to create initially + type: org.apache.brooklyn.api.entity.EntitySpec + default: + $brooklyn:entitySpec: + type: kubernetes-default-pods brooklyn.enrichers: - type: org.apache.brooklyn.enricher.stock.Propagator @@ -340,24 +348,14 @@ brooklyn.catalog: sudo -E calicoctl pool add ${FLANNEL_NETWORK} --nat-outgoing checkRunning.command: | sudo -E calicoctl pool show --ipv4 | grep "${FLANNEL_NETWORK}" - - type: empty-software-process + - type: conditional-entity id: kubernetes-pods name: "kubernetes-pods" brooklyn.config: + conditional.entity.create: true + conditional.entity.propagate: false + conditional.entity.spec: $brooklyn:entity("kubernetes-cluster").config("kubernetes.pods.spec") start.latch: $brooklyn:entity("kubernetes-pods").sibling("kube-apiserver").attributeWhenReady("service.isUp") - brooklyn.children: - - type: calico-policy-controller-pod - id: calico-policy-controller - name: "calico-policy-controller" - - type: kube-dns-pod - id: kube-dns - name: "kube-dns" - - type: kubernetes-dashboard-pod - id: kubernetes-dashboard - name: "kubernetes-dashboard" - - type: prometheus-pod - id: prometheus - name: "prometheus" dynamiccluster.memberspec: $brooklyn:entitySpec: type: kubernetes-master diff --git a/kubernetes/catalog/kubernetes/pods.bom b/kubernetes/catalog/kubernetes/pods.bom index 9d099897..0cba6e55 100644 --- a/kubernetes/catalog/kubernetes/pods.bom +++ b/kubernetes/catalog/kubernetes/pods.bom @@ -193,3 +193,27 @@ brooklyn.catalog: template.substitutions: kubernetes_url: $brooklyn:entity("kubernetes-cluster").attributeWhenReady("kubernetes.url") + + - id: kubernetes-default-pods + name: "Kubernetes Default Pods" + description: | + Default set of Kubernetes pods to launch at startup + itemType: entity + item: + type: empty-software-process + id: kubernetes-default-pods + name: "kubernetes-default-pods" + + brooklyn.children: + - type: calico-policy-controller-pod + id: calico-policy-controller + name: "calico-policy-controller" + - type: kube-dns-pod + id: kube-dns + name: "kube-dns" + - type: kubernetes-dashboard-pod + id: kubernetes-dashboard + name: "kubernetes-dashboard" + - type: prometheus-pod + id: prometheus + name: "prometheus" From 98aa04c626b57cea2abffa0d4acee5b10bb59721 Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Tue, 6 Dec 2016 00:46:36 +0000 Subject: [PATCH 2/5] Use master in entity names and include load balancer in hosts file --- kubernetes/catalog/kubernetes/kubernetes.bom | 31 +++++++++----------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/kubernetes/catalog/kubernetes/kubernetes.bom b/kubernetes/catalog/kubernetes/kubernetes.bom index 9c7fa489..d3e3f598 100644 --- a/kubernetes/catalog/kubernetes/kubernetes.bom +++ b/kubernetes/catalog/kubernetes/kubernetes.bom @@ -135,24 +135,21 @@ brooklyn.catalog: factoryMethod.args: - $brooklyn:attributeWhenReady("application.id") - $brooklyn:object: - type: com.google.common.base.Predicates - factoryMethod.name: "or" + type: org.apache.brooklyn.core.entity.EntityPredicates + factoryMethod.name: "displayNameSatisfies" factoryMethod.args: - $brooklyn:object: - type: org.apache.brooklyn.core.entity.EntityPredicates - factoryMethod.name: "displayNameEqualTo" - factoryMethod.args: - - "kubernetes-master" - - $brooklyn:object: - type: org.apache.brooklyn.core.entity.EntityPredicates - factoryMethod.name: "displayNameEqualTo" + type: com.google.common.base.Predicates + factoryMethod.name: "in" factoryMethod.args: - - "kubernetes-worker" + - [ "kubernetes-master", "kubernetes-worker", kubernetes-master-load-balancer ] - id: kubernetes-cluster name: "Kubernetes Cluster" description: | - Creates a Kubernetes cluster with a manager and a configurable number of workers. + Creates a Kubernetes cluster with a configurable number of masters and workers + and an HAProxy load balancer for the masters. + Requires a pre-existing discovery mechanism and references to a CA server entity. itemType: entity item: @@ -253,15 +250,15 @@ brooklyn.catalog: - type: org.apache.brooklyn.enricher.stock.Propagator brooklyn.config: uniqueTag: kubernetes-url-and-endpoint-propagator - producer: $brooklyn:entity("kubernetes-manager-load-balancer") + producer: $brooklyn:entity("kubernetes-master-load-balancer") propagating: - $brooklyn:sensor("kubernetes.url") - $brooklyn:sensor("kubernetes.endpoint") brooklyn.children: - type: haproxy-load-balancer - id: kubernetes-manager-load-balancer - name: "kubernetes-manager-load-balancer" + id: kubernetes-master-load-balancer + name: "kubernetes-master-load-balancer" brooklyn.config: haproxy.port: $brooklyn:parent().parent().config("kubernetes.apiserver.port") haproxy.protocol: "http" @@ -319,7 +316,7 @@ brooklyn.catalog: - type: org.apache.brooklyn.enricher.stock.Propagator brooklyn.config: uniqueTag: kubernetes-url-propagator - producer: $brooklyn:entity("kubernetes-manager-load-balancer") + producer: $brooklyn:entity("kubernetes-master-load-balancer") propagating: - $brooklyn:sensor("kubernetes.url") brooklyn.config: @@ -332,7 +329,7 @@ brooklyn.catalog: id: kubernetes-master name: "kubernetes-master" brooklyn.config: - install.latch: $brooklyn:entity("kubernetes-manager-load-balancer").attributeWhenReady("service.isUp") + install.latch: $brooklyn:entity("kubernetes-master-load-balancer").attributeWhenReady("service.isUp") launch.latch: $brooklyn:entity("etcd-cluster").attributeWhenReady("service.isUp") kubernetes.schedulable: false brooklyn.children: @@ -362,7 +359,7 @@ brooklyn.catalog: id: kubernetes-master name: "kubernetes-master" brooklyn.config: - install.latch: $brooklyn:entity("kubernetes-manager-load-balancer").attributeWhenReady("service.isUp") + install.latch: $brooklyn:entity("kubernetes-master-load-balancer").attributeWhenReady("service.isUp") launch.latch: $brooklyn:entity("etcd-cluster").attributeWhenReady("service.isUp") kubernetes.schedulable: false From eb98a297e3a0ad39e568180ebd627c2276fd5e3c Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Tue, 6 Dec 2016 00:47:00 +0000 Subject: [PATCH 3/5] Export contents of kubeconfig YAML file as sensor --- kubernetes/catalog/kubernetes/kubernetes.bom | 29 +++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/kubernetes/catalog/kubernetes/kubernetes.bom b/kubernetes/catalog/kubernetes/kubernetes.bom index d3e3f598..47fae602 100644 --- a/kubernetes/catalog/kubernetes/kubernetes.bom +++ b/kubernetes/catalog/kubernetes/kubernetes.bom @@ -184,7 +184,7 @@ brooklyn.catalog: - name: kubernetes.cluster.name label: "Kubernetes Cluster Name" type: string - default: "amp" + default: "clocker" - name: kubernetes.version label: "Kubernetes Version" type: string @@ -254,6 +254,12 @@ brooklyn.catalog: propagating: - $brooklyn:sensor("kubernetes.url") - $brooklyn:sensor("kubernetes.endpoint") + - type: org.apache.brooklyn.enricher.stock.Propagator + brooklyn.config: + uniqueTag: kubeconfig-yaml-propagator + producer: $brooklyn:entity("kubernetes-master-cluster") + propagating: + - $brooklyn:sensor("kubernetes.kubeconfig.yaml") brooklyn.children: - type: haproxy-load-balancer @@ -319,6 +325,20 @@ brooklyn.catalog: producer: $brooklyn:entity("kubernetes-master-load-balancer") propagating: - $brooklyn:sensor("kubernetes.url") + - type: org.apache.brooklyn.enricher.stock.Aggregator + brooklyn.config: + uniqueTag: kubeconfig-yaml-aggregator + enricher.sourceSensor: $brooklyn:sensor("kubernetes.kubeconfig.yaml") + enricher.targetSensor: $brooklyn:sensor("kubernetes.kubeconfig.list") + enricher.aggregating.fromMembers: true + - type: org.apache.brooklyn.enricher.stock.Joiner + brooklyn.config: + uniqueTag: kubeconfig-yaml-joiner + enricher.sourceSensor: $brooklyn:sensor("kubernetes.kubeconfig.list") + enricher.targetSensor: $brooklyn:sensor("kubernetes.kubeconfig.yaml") + enricher.joiner.quote: false + enricher.joiner.minimum: 1 + enricher.joiner.maximum: 1 brooklyn.config: cluster.initial.size: $brooklyn:entity("kubernetes-cluster").config("kubernetes.master.size") dynamiccluster.quarantineFailedEntities: @@ -630,6 +650,13 @@ brooklyn.catalog: command: | kubectl get node ${HOST_SUBNET_ADDRESS} -o jsonpath='{range @.status.conditions[*]}{@.type}={@.status} ' | grep True | cut -d= -f1 + - type: org.apache.brooklyn.core.sensor.ssh.SshCommandSensor + brooklyn.config: + name: kubernetes.kubeconfig.yaml + description: | + Returns contents of a kubeconfig YAML file + command: | + kubectl config view --output=yaml --flatten=true --minify=true - type: org.apache.brooklyn.core.effector.ssh.SshCommandEffector brooklyn.config: name: kubectl From fd583b9d70a6417de067da1c6c9431f6a9863716 Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Tue, 6 Dec 2016 01:08:41 +0000 Subject: [PATCH 4/5] Do not use template.substitutions in pod resources --- kubernetes/catalog/kubernetes/pods.bom | 25 +++++++------------ kubernetes/resources/kubernetes/kube-dns.yaml | 14 +++++------ .../kubernetes/kubernetes-dashboard.yaml | 2 +- .../kubernetes/policy-controller.yaml | 4 +-- .../resources/kubernetes/prometheus.yaml | 8 +++--- 5 files changed, 23 insertions(+), 30 deletions(-) diff --git a/kubernetes/catalog/kubernetes/pods.bom b/kubernetes/catalog/kubernetes/pods.bom index 0cba6e55..28e696c0 100644 --- a/kubernetes/catalog/kubernetes/pods.bom +++ b/kubernetes/catalog/kubernetes/pods.bom @@ -26,9 +26,6 @@ brooklyn.catalog: - name: kubernetes.pod.name label: "Kubernetes Pod Name" type: string - - name: template.substitutions - label: "Template Substitutions" - type: java.util.Map brooklyn.config: install.unique_label: @@ -135,11 +132,10 @@ brooklyn.catalog: kubernetes.pod.file: "classpath://io.brooklyn.clocker.kubernetes:kubernetes/kube-dns.yaml" kubernetes.pod.namespace: "kube-system" - template.substitutions: - kubernetes_url: $brooklyn:entity("kubernetes-cluster").attributeWhenReady("kubernetes.url") - replicas: $brooklyn:entity("kubernetes-cluster").config("kubernetes.initial.size") - dns_service_ip: $brooklyn:entity("kubernetes-cluster").config("kubernetes.dns.address") - dns_service_domain: $brooklyn:entity("kubernetes-cluster").config("kubernetes.dns.domain") + kubernetes.url: $brooklyn:entity("kubernetes-cluster").attributeWhenReady("kubernetes.url") + dns.replicas: $brooklyn:entity("kubernetes-cluster").config("kubernetes.initial.size") + dns.service.ip: $brooklyn:entity("kubernetes-cluster").config("kubernetes.dns.address") + dns.service.domain: $brooklyn:entity("kubernetes-cluster").config("kubernetes.dns.domain") - id: kubernetes-dashboard-pod name: "Kubernetes Dashboard Pod" @@ -155,8 +151,7 @@ brooklyn.catalog: kubernetes.pod.file: "classpath://io.brooklyn.clocker.kubernetes:kubernetes/kubernetes-dashboard.yaml" kubernetes.pod.namespace: "kube-system" - template.substitutions: - kubernetes_url: $brooklyn:entity("kubernetes-cluster").attributeWhenReady("kubernetes.url") + kubernetes.url: $brooklyn:entity("kubernetes-cluster").attributeWhenReady("kubernetes.url") - id: calico-policy-controller-pod name: "Calico Policy Controller Pod" @@ -172,9 +167,8 @@ brooklyn.catalog: kubernetes.pod.file: "classpath://io.brooklyn.clocker.kubernetes:kubernetes/policy-controller.yaml" kubernetes.pod.namespace: "kube-system" - template.substitutions: - kubernetes_url: $brooklyn:entity("kubernetes-cluster").attributeWhenReady("kubernetes.url") - etcd_endpoints: $brooklyn:entity("etcd-cluster").attributeWhenReady("etcd.urls") + kubernetes.url: $brooklyn:entity("kubernetes-cluster").attributeWhenReady("kubernetes.url") + etcd.endpoints: $brooklyn:entity("etcd-cluster").attributeWhenReady("etcd.urls") - id: prometheus-pod name: "Prometheus Monitoring Pod" @@ -191,13 +185,12 @@ brooklyn.catalog: kubernetes.pod.file: "classpath://io.brooklyn.clocker.kubernetes:kubernetes/prometheus.yaml" kubernetes.pod.namespace: "kube-system" - template.substitutions: - kubernetes_url: $brooklyn:entity("kubernetes-cluster").attributeWhenReady("kubernetes.url") + kubernetes.url: $brooklyn:entity("kubernetes-cluster").attributeWhenReady("kubernetes.url") - id: kubernetes-default-pods name: "Kubernetes Default Pods" description: | - Default set of Kubernetes pods to launch at startup + Default set of Kubernetes pods to launch at startup itemType: entity item: type: empty-software-process diff --git a/kubernetes/resources/kubernetes/kube-dns.yaml b/kubernetes/resources/kubernetes/kube-dns.yaml index 0b98cf2b..06b740f7 100644 --- a/kubernetes/resources/kubernetes/kube-dns.yaml +++ b/kubernetes/resources/kubernetes/kube-dns.yaml @@ -11,7 +11,7 @@ metadata: spec: selector: app: kube-dns - clusterIP: ${config['template.substitutions']['dns_service_ip']} + clusterIP: ${config['dns.service.ip']} ports: - name: dns port: 53 @@ -31,7 +31,7 @@ metadata: version: v20 kubernetes.io/cluster-service: "true" spec: - replicas: ${config['template.substitutions']['replicas']} + replicas: ${config['dns.replicas']} selector: app: kube-dns version: v20 @@ -72,9 +72,9 @@ spec: initialDelaySeconds: 30 timeoutSeconds: 5 args: - - --domain=${config['template.substitutions']['dns_service_domain']}.local. + - --domain=${config['dns.service.domain']}.local. - --dns-port=10053 - - --kube-master-url=${config['template.substitutions']['kubernetes_url']} + - --kube-master-url=${config['kubernetes.url']} ports: - containerPort: 10053 name: dns-local @@ -106,11 +106,11 @@ spec: memory: 50Mi args: - >- - -cmd=nslookup kubernetes.default.svc.${config['template.substitutions']['dns_service_domain']}.local 127.0.0.1 > /dev/null && - nslookup kubernetes.default.svc.${config['template.substitutions']['dns_service_domain']}.local 127.0.0.1:10053 > /dev/null + -cmd=nslookup kubernetes.default.svc.${config['dns.service.domain']}.local 127.0.0.1 > /dev/null && + nslookup kubernetes.default.svc.${config['dns.service.domain']}.local 127.0.0.1:10053 > /dev/null - -port=8080 - -quiet ports: - containerPort: 8080 protocol: TCP - dnsPolicy: Default \ No newline at end of file + dnsPolicy: Default diff --git a/kubernetes/resources/kubernetes/kubernetes-dashboard.yaml b/kubernetes/resources/kubernetes/kubernetes-dashboard.yaml index 1143d3a5..cbc5cb00 100644 --- a/kubernetes/resources/kubernetes/kubernetes-dashboard.yaml +++ b/kubernetes/resources/kubernetes/kubernetes-dashboard.yaml @@ -28,7 +28,7 @@ spec: - containerPort: 9090 protocol: TCP args: - - --apiserver-host=${config['template.substitutions']['kubernetes_url']} + - --apiserver-host=${config['kubernetes.url']} livenessProbe: httpGet: path: / diff --git a/kubernetes/resources/kubernetes/policy-controller.yaml b/kubernetes/resources/kubernetes/policy-controller.yaml index 69992f85..472b2311 100644 --- a/kubernetes/resources/kubernetes/policy-controller.yaml +++ b/kubernetes/resources/kubernetes/policy-controller.yaml @@ -29,9 +29,9 @@ spec: image: calico/kube-policy-controller:v0.3.0 env: - name: ETCD_ENDPOINTS - value: "${config['template.substitutions']['etcd_endpoints']}" + value: "${config['etcd.endpoints']}" - name: K8S_API - value: "${config['template.substitutions']['kubernetes_url']}" + value: "${config['kubernetes.url']}" - name: LEADER_ELECTION value: "true" - name: CONFIGURE_ETC_HOSTS diff --git a/kubernetes/resources/kubernetes/prometheus.yaml b/kubernetes/resources/kubernetes/prometheus.yaml index 7ce5ca59..9da52e36 100644 --- a/kubernetes/resources/kubernetes/prometheus.yaml +++ b/kubernetes/resources/kubernetes/prometheus.yaml @@ -83,7 +83,7 @@ data: - job_name: 'kubernetes-cluster' kubernetes_sd_configs: - api_servers: - - ${config['template.substitutions']['kubernetes_url']} + - ${config['kubernetes.url']} basic_auth: username: prometheus password: pr0m3th3us @@ -91,7 +91,7 @@ data: - job_name: 'kubernetes-nodes' kubernetes_sd_configs: - api_servers: - - ${config['template.substitutions']['kubernetes_url']} + - ${config['kubernetes.url']} basic_auth: username: prometheus password: pr0m3th3us @@ -109,7 +109,7 @@ data: - job_name: 'kubernetes-service-endpoints' kubernetes_sd_configs: - api_servers: - - ${config['template.substitutions']['kubernetes_url']} + - ${config['kubernetes.url']} basic_auth: username: prometheus password: pr0m3th3us @@ -138,4 +138,4 @@ data: target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_service_name] action: replace - target_label: kubernetes_name \ No newline at end of file + target_label: kubernetes_name From 5acecd63cc31091b9c55cd7e0a99f1de391f5cc5 Mon Sep 17 00:00:00 2001 From: Andrew Donald Kennedy Date: Tue, 6 Dec 2016 01:23:35 +0000 Subject: [PATCH 5/5] Move config of VXLAN as default to Flannel entity --- kubernetes/catalog/kubernetes/kubernetes.bom | 1 - kubernetes/catalog/kubernetes/plugins.bom | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/kubernetes/catalog/kubernetes/kubernetes.bom b/kubernetes/catalog/kubernetes/kubernetes.bom index 47fae602..50a56c59 100644 --- a/kubernetes/catalog/kubernetes/kubernetes.bom +++ b/kubernetes/catalog/kubernetes/kubernetes.bom @@ -563,7 +563,6 @@ brooklyn.catalog: brooklyn.config: etcd.endpoints: $brooklyn:entity("etcd-cluster").attributeWhenReady("etcd.urls") flannel.network: $brooklyn:entity("kubernetes-cluster").config("kubernetes.pod.cidr") - flannel.transport: "vxlan" - type: docker-engine-with-resilience id: docker-engine name: "docker-engine" diff --git a/kubernetes/catalog/kubernetes/plugins.bom b/kubernetes/catalog/kubernetes/plugins.bom index 72e4ccb5..3320707c 100644 --- a/kubernetes/catalog/kubernetes/plugins.bom +++ b/kubernetes/catalog/kubernetes/plugins.bom @@ -42,7 +42,7 @@ brooklyn.catalog: The Flannel backend transport mechanism. Can be any of: 'udp', 'vxlan', 'host-gw', 'aws-vpc', 'gce' or 'alloc'. type: string - default: "udp" + default: "vxlan" - name: flannel.interface label: "Flannel Interface" description: |