[WIP] Configure TLS for Kubernetes services

kubernetes/catalog/kubernetes/pods.bom

@@ -102,6 +102,14 @@ brooklyn.catalog:
               kubectl get pod --namespace=${NAMESPACE} -l app=${NAME} --output=jsonpath='{range .items[0]}{.status.phase}'
 
       brooklyn.enrichers:
+        - type: org.apache.brooklyn.core.entity.lifecycle.ServiceStateLogic$ComputeServiceIndicatorsFromChildrenAndMembers
+          brooklyn.config:
+            uniqueTag: pod-service-up-logic
+            enricher.service_state.children_and_members.quorum.up: "alwaysHealthy"
+            enricher.service_state.children_and_members.quorum.running: "alwaysHealthy"
+            enricher.aggregating.fromChildren: true
+            enricher.aggregating.fromMembers: false
+            enricher.suppressDuplicates: true
         - type: org.apache.brooklyn.enricher.stock.Transformer
           brooklyn.config:
             uniqueTag: kubernetes-pod-url-publisher

kubernetes/examples/kubernetes.yaml

@@ -10,7 +10,7 @@ location:
     region: ams01
     identity: xxxxxxxx
     credential: XXXXXXXX
-    privateKeyFile: ~/.ssh/softlayer.pem
+    privateKeyFile: "~/.ssh/softlayer.pem"
     customizers:
       - $brooklyn:object:
           type: org.apache.brooklyn.location.jclouds.softlayer.SoftLayerSameVlanLocationCustomizer
@@ -23,13 +23,14 @@ services:
     name: "kubernetes"
     brooklyn.config:
       kubernetes.debug: true
-      kubernetes.version: 1.4.3
+      kubernetes.version: 1.4.5
       start.timeout: 30m
       kubernetes.master.size: 2
       kubernetes.initial.size: 4
       kubernetes.max.size: 16
       etcd.initial.size: 3
-      kubernetes.apiserver.port: 8000
+      kubernetes.apiserver.port: 8443
+      kubernetes.apiserver.protocol: "https"
       kubernetes.scaling.cpu.limit: 0.80
       kubernetes.recovery.stabilizationDelay: 30s
       kubernetes.recovery.failOnRecurringFailuresInThisDuration: 300000

kubernetes/resources/kubernetes/known_tokens.csv

@@ -0,0 +1 @@
+DwYThAUb63Mu4au35XHnbe3Y3cTu5u8j,admin,admin

kubernetes/resources/kubernetes/kube-dns.yaml

@@ -1,13 +1,13 @@
-apiVersion: v1
 kind: Service
+apiVersion: v1
 metadata:
   name: kube-dns
   namespace: kube-system
   labels:
     app: kube-dns
     name: kube-dns-service
     kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "KubeDNS"
+    kubernetes.io/name: "kube-dns"
 spec:
   selector:
     app: kube-dns
@@ -30,6 +30,7 @@ metadata:
     name: kube-dns-replicationcontroller
     version: v20
     kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "kube-dns"
 spec:
   replicas: ${config['template.substitutions']['replicas']}
   selector:
@@ -71,10 +72,14 @@ spec:
             scheme: HTTP
           initialDelaySeconds: 30
           timeoutSeconds: 5
+        volumeMounts:
+          - mountPath: "/etc/kubernetes/kubeconfig"
+            name: kubeconfig
+            readOnly: true
         args:
           - --domain=${config['template.substitutions']['dns_service_domain']}.local.
           - --dns-port=10053
-          - --kube-master-url=${config['template.substitutions']['kubernetes_url']}
+          - --kubecfg-file=/etc/kubernetes/kubeconfig
         ports:
           - containerPort: 10053
             name: dns-local
@@ -113,4 +118,8 @@ spec:
         ports:
           - containerPort: 8080
             protocol: TCP
-      dnsPolicy: Default
+    volumes:
+      - name: kubeconfig
+        hostPath:
+          path: "/etc/kubernetes/kubeconfig"
+    dnsPolicy: Default

kubernetes/resources/kubernetes/kubernetes-dashboard.yaml

@@ -7,34 +7,48 @@ metadata:
     app: kubernetes-dashboard
     name: kubernetes-dashboard-deployment
     version: v1.4.1
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "kubernetes-dashboard"
 spec:
   replicas: 1
   selector:
     matchLabels:
       app: kubernetes-dashboard
+      kubernetes.io/cluster-service: "true"
   template:
     metadata:
       labels:
         app: kubernetes-dashboard
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
         scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+        version: v1.4.1
+        kubernetes.io/cluster-service: "true"
     spec:
       containers:
       - name: kubernetes-dashboard
         image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.4.1
         imagePullPolicy: Always
         ports:
-        - containerPort: 9090
-          protocol: TCP
-        args:
-        - --apiserver-host=${config['template.substitutions']['kubernetes_url']}
+          - containerPort: 9090
+            protocol: TCP
+        volumeMounts:
+          - mountPath: "/etc/kubernetes/kubeconfig"
+            name: kubeconfig
+            readOnly: true
+        env:
+          - name: KUBECONFIG
+            value: "/etc/kubernetes/kubeconfig"
         livenessProbe:
           httpGet:
             path: /
             port: 9090
           initialDelaySeconds: 30
           timeoutSeconds: 30
+      volumes:
+        - name: kubeconfig
+          hostPath:
+            path: "/etc/kubernetes/kubeconfig"
 ---
 kind: Service
 apiVersion: v1
@@ -47,7 +61,7 @@ metadata:
 spec:
   type: NodePort
   ports:
-  - port: 80
-    targetPort: 9090
+    - port: 80
+      targetPort: 9090
   selector:
     app: kubernetes-dashboard

kubernetes/resources/kubernetes/policy.jsonl

@@ -0,0 +1,4 @@
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user": "admin", "namespace": "*", "resource": "*", "apiGroup": "*"}}
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user": "kubelet", "namespace": "*", "resource": "pods", "readonly": true}}
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user": "kubelet", "namespace": "*", "resource": "events"}}
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user": "*", "readonly": true, "nonResourcePath": "*"}}

kubernetes/resources/kubernetes/prometheus.yaml

@@ -7,7 +7,7 @@ metadata:
     app: prometheus
     name: prometheus-service
   annotations:
-    prometheus.io/scrape: 'true'
+    prometheus.io/scrape: "true"
 spec:
   selector:
     app: prometheus
@@ -26,12 +26,20 @@ metadata:
   labels:
     app: prometheus
     name: prometheus-deployment
+    version: v1.1.3
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "prometheus"
 spec:
   replicas: 1
+  selector:
+    app: prometheus
+    kubernetes.io/cluster-service: "true"
   template:
     metadata:
       labels:
         app: prometheus
+        version: v1.1.3
+        kubernetes.io/cluster-service: "true"
     spec:
       containers:
       - image: quay.io/prometheus/prometheus:v1.1.3
@@ -65,8 +73,8 @@ spec:
           configMap:
             name: prometheus-config
 ---
-apiVersion: v1
 kind: ConfigMap
+apiVersion: v1
 metadata:
   name: prometheus-config
   namespace: kube-system
@@ -81,38 +89,37 @@ data:
           - targets:
               - localhost:9090
       - job_name: 'kubernetes-cluster'
+        scheme: https
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
         kubernetes_sd_configs:
           - api_servers:
-              - ${config['template.substitutions']['kubernetes_url']}
-            basic_auth:
-              username: prometheus
-              password: pr0m3th3us
+              - https://kubernetes.default.svc
+            in_cluster: true
             role: apiserver
       - job_name: 'kubernetes-nodes'
+        scheme: https
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
         kubernetes_sd_configs:
           - api_servers:
-              - ${config['template.substitutions']['kubernetes_url']}
-            basic_auth:
-              username: prometheus
-              password: pr0m3th3us
+              - https://kubernetes.default.svc
+            in_cluster: true
             role: node
         relabel_configs:
           - action: labelmap
             regex: __meta_kubernetes_node_label_(.+)
-          - source_labels: [__meta_kubernetes_role]
-            action: replace
-            target_label: kubernetes_role
-          - source_labels: [__address__]
-            regex: (.*):10250
-            replacement: ${r"${1}:10255"}
-            target_label: __address__
       - job_name: 'kubernetes-service-endpoints'
+        scheme: https
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
         kubernetes_sd_configs:
           - api_servers:
-              - ${config['template.substitutions']['kubernetes_url']}
-            basic_auth:
-              username: prometheus
-              password: pr0m3th3us
+              - https://kubernetes.default.svc
+            in_cluster: true
             role: endpoint
         relabel_configs:
           - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
@@ -138,4 +145,4 @@ data:
             target_label: kubernetes_namespace
           - source_labels: [__meta_kubernetes_service_name]
             action: replace
-            target_label: kubernetes_name
+            target_label: kubernetes_name