From d3d638dffec8d6488c5d296a4dba27bd43af18e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Okouya?= Date: Sun, 22 Jul 2018 16:54:05 +0200 Subject: [PATCH 1/7] adding various options to conform to v1.11.1 cmd line option for component such as kubelet, apiserver, scheduler, kubeproxy --- clustervars.yml | 11 +++++++++- provisioning/group_vars/all.yml | 9 -------- .../apiserver/defaults/main.yml | 12 ++++++----- .../scheduler/templates/scheduler-config.yml | 6 ++++++ .../templates/scheduler.systemd.service | 13 ++++++++++++ .../templates/kube-proxy.systemd.service | 12 +++++++++++ .../rok.kube-node/kubelet/defaults/main.yml | 8 +++---- .../kubelet/templates/kubelet-config.yml | 18 ++++++++++++++++ .../kubelet/templates/kubelet-systemd.service | 21 +++++++++++++++++++ 9 files changed, 91 insertions(+), 19 deletions(-) create mode 100644 provisioning/roles/rok.kube-controlplane/scheduler/templates/scheduler-config.yml create mode 100644 provisioning/roles/rok.kube-controlplane/scheduler/templates/scheduler.systemd.service create mode 100644 provisioning/roles/rok.kube-node/kube-proxy/templates/kube-proxy.systemd.service create mode 100644 provisioning/roles/rok.kube-node/kubelet/templates/kubelet-config.yml create mode 100644 provisioning/roles/rok.kube-node/kubelet/templates/kubelet-systemd.service diff --git a/clustervars.yml b/clustervars.yml index 4516b61..42c0b90 100644 --- a/clustervars.yml +++ b/clustervars.yml @@ -40,7 +40,7 @@ vagrant: cluster: name: "local" domain: "cluster.local" - kubernetes_version: 'v1.10.0' + kubernetes_version: 'v1.11.1' networking: pod_cidr_address: "172.16.0.0/16" service_cidr_address: "172.20.0.0/16" @@ -54,6 +54,15 @@ ingress: vagrant_enabled: false nodename: node02 +kubeapps: + dashboard: + enbaled: false + traefik: + enaled: true + spinnaker: + enabled: true + + ### -- kubernetes / ansible extra vars / vagrant vmbox variables server: etcd: ## etcd is for ansible only. etcd is on same vm as master diff --git a/provisioning/group_vars/all.yml b/provisioning/group_vars/all.yml index 3e2c7f6..d9e0d66 100644 --- a/provisioning/group_vars/all.yml +++ b/provisioning/group_vars/all.yml @@ -2,17 +2,12 @@ etcd_version: 'v3.2.11' kube_version: "{{ cluster.kubernetes_version | default('v1.10.0') }}" -helm_version: 'helm-v2.8.0' -istio_version: '0.7.1' platform_arch: "{{ (ansible_architecture == 'armv6l') | ternary('aarch','amd64') }}" kube_mirror_url: 'https://storage.googleapis.com/kubernetes-release/release' etcd_mirror_url: 'https://storage.googleapis.com/etcd' istio_mirror_url: 'https://storage.googleapis.com/istio-release/releases' -helm_mirror_url: 'https://storage.googleapis.com/kubernetes-helm' -helm_pkg_sha256sum: "{% if ansible_system == 'Darwin' %}{{ sha256_darwin }}{% else %}{{ sha256_linux }}{% endif %}" - etcd_cluster_list: "https://{{ server.etcd.nodes[0].nodename }}:2379" kube_master_apiserver_count: "{{ groups['controlplane'] | length }}" @@ -89,7 +84,3 @@ images: traefik: name: traefik version: v1.6-alpine - - haproxy: - name: quay.io/jcmoraisjr/haproxy-ingress - version: v0.5-beta.1 diff --git a/provisioning/roles/rok.kube-controlplane/apiserver/defaults/main.yml b/provisioning/roles/rok.kube-controlplane/apiserver/defaults/main.yml index df7f474..dcbf352 100644 --- a/provisioning/roles/rok.kube-controlplane/apiserver/defaults/main.yml +++ b/provisioning/roles/rok.kube-controlplane/apiserver/defaults/main.yml @@ -9,13 +9,13 @@ admission_plugins: - NamespaceLifecycle - LimitRanger - ServiceAccount - - PersistentVolumeLabel - - DefaultStorageClass - - ResourceQuota - NodeRestriction + - DefaultStorageClass - DefaultTolerationSeconds + - PersistentVolumeLabel - MutatingAdmissionWebhook - ValidatingAdmissionWebhook + - ResourceQuota - Priority #--enable-admission-plugins @@ -28,7 +28,7 @@ apiserver_opts: "profiling": "false" "bind-address": "0.0.0.0" "anonymous-auth": "false" - "client-ca-file": /etc/kubernetes/certs/ca.pem + "client-ca-file": "/etc/kubernetes/certs/ca.pem" "enable-swagger-ui": "true" "etcd-cafile": /etc/kubernetes/certs/ca.pem "etcd-certfile": /etc/kubernetes/certs/etcd-client.pem @@ -37,9 +37,10 @@ apiserver_opts: "event-ttl": "1h" "external-hostname": "{{ server.controlplane.nodes[0].fqdn }}" "secure-port": "{{ kube_master_secure_port }}" + "kubelet-certificate-authority": "/etc/kubernetes/certs/ca.pem" "kubelet-client-certificate": "/etc/kubernetes/certs/apiserver-kubelet-client.pem" "kubelet-client-key": "/etc/kubernetes/certs/apiserver-kubelet-client-key.pem" - "kubelet-preferred-address-types": "InternalDNS,InternalIP,Hostname,ExternalIP,ExternalDNS" + "kubelet-preferred-address-types": "InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP" "runtime-config": "extensions/v1beta1=true,extensions/v1beta1/networkpolicies=true,authentication.k8s.io/v1beta1=true" "service-account-lookup": "true" "service-account-key-file": /etc/kubernetes/certs/apiserver.pem @@ -53,6 +54,7 @@ apiserver_opts: +## This was for kubernetes <= 1.9 api # apiserver_opts: # "admission-control": "{{ admission_plugins | join(',') }}" # "advertise-address": "{{ prefered_iface }}" diff --git a/provisioning/roles/rok.kube-controlplane/scheduler/templates/scheduler-config.yml b/provisioning/roles/rok.kube-controlplane/scheduler/templates/scheduler-config.yml new file mode 100644 index 0000000..b7b6a75 --- /dev/null +++ b/provisioning/roles/rok.kube-controlplane/scheduler/templates/scheduler-config.yml @@ -0,0 +1,6 @@ +apiVersion: componentconfig/v1alpha1 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "/var/lib/kubernetes/kube-scheduler.kubeconfig" +leaderElection: + leaderElect: true \ No newline at end of file diff --git a/provisioning/roles/rok.kube-controlplane/scheduler/templates/scheduler.systemd.service b/provisioning/roles/rok.kube-controlplane/scheduler/templates/scheduler.systemd.service new file mode 100644 index 0000000..251b13d --- /dev/null +++ b/provisioning/roles/rok.kube-controlplane/scheduler/templates/scheduler.systemd.service @@ -0,0 +1,13 @@ +[Unit] +Description=Kubernetes Scheduler +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +ExecStart=/usr/local/bin/kube-scheduler \\ + --config=/etc/kubernetes/config/kube-scheduler.yaml \\ + --v=2 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/provisioning/roles/rok.kube-node/kube-proxy/templates/kube-proxy.systemd.service b/provisioning/roles/rok.kube-node/kube-proxy/templates/kube-proxy.systemd.service new file mode 100644 index 0000000..7379596 --- /dev/null +++ b/provisioning/roles/rok.kube-node/kube-proxy/templates/kube-proxy.systemd.service @@ -0,0 +1,12 @@ +[Unit] +Description=Kubernetes Kube Proxy +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +ExecStart=/usr/local/bin/kube-proxy \\ + --config=/var/lib/kube-proxy/kube-proxy-config.yaml +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/provisioning/roles/rok.kube-node/kubelet/defaults/main.yml b/provisioning/roles/rok.kube-node/kubelet/defaults/main.yml index f50cb2b..bedc741 100644 --- a/provisioning/roles/rok.kube-node/kubelet/defaults/main.yml +++ b/provisioning/roles/rok.kube-node/kubelet/defaults/main.yml @@ -5,7 +5,7 @@ kubelet_schedulable: '{% if "worker" in group_names %}true{% else %}false{% endi kubelet_master_only: '{% if inventory_hostname in groups["controlplane"] and inventory_hostname not in groups["worker"] %}true{% else %}false{% endif %}' kubelet_overrides: - 'fail-swap-on': 'false' + 'fail-swap-on': 'true' kubelet_opts: 'allow-privileged': 'true' @@ -24,9 +24,9 @@ kubelet_opts: 'alsologtostderr': 'true' 'fail-swap-on': 'false' 'kubeconfig': '{{ kube_config.kubelet }}' - 'node-labels': '{% if "controlplane" in group_names %},node-role.kubernetes.io/master= - {%- if not kubelet_master_only|bool %},node-role.kubernetes.io/node={% endif %} - {%- else %}node-role.kubernetes.io/node= + 'node-labels': '{% if "controlplane" in group_names %},node-role.kubernetes.io/master + {%- if not kubelet_master_only|bool %},node-role.kubernetes.io/node{% endif %} + {%- else %}node-role.kubernetes.io/node {%- endif %}' 'node-ip': '{{ prefered_iface }}' 'pod-infra-container-image': '{{ images.pause.name }}:{{ images.pause.version }}' diff --git a/provisioning/roles/rok.kube-node/kubelet/templates/kubelet-config.yml b/provisioning/roles/rok.kube-node/kubelet/templates/kubelet-config.yml new file mode 100644 index 0000000..7ac5bf6 --- /dev/null +++ b/provisioning/roles/rok.kube-node/kubelet/templates/kubelet-config.yml @@ -0,0 +1,18 @@ +kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +authentication: + anonymous: + enabled: false + webhook: + enabled: true + x509: + clientCAFile: "/var/lib/kubernetes/ca.pem" +authorization: + mode: Webhook +clusterDomain: "cluster.local" +clusterDNS: + - "10.32.0.10" +podCIDR: "${POD_CIDR}" +runtimeRequestTimeout: "15m" +tlsCertFile: "/var/lib/kubelet/${HOSTNAME}.pem" +tlsPrivateKeyFile: "/var/lib/kubelet/${HOSTNAME}-key.pem" \ No newline at end of file diff --git a/provisioning/roles/rok.kube-node/kubelet/templates/kubelet-systemd.service b/provisioning/roles/rok.kube-node/kubelet/templates/kubelet-systemd.service new file mode 100644 index 0000000..f5fb742 --- /dev/null +++ b/provisioning/roles/rok.kube-node/kubelet/templates/kubelet-systemd.service @@ -0,0 +1,21 @@ +[Unit] +Description=Kubernetes Kubelet +Documentation=https://github.com/kubernetes/kubernetes +After=containerd.service +Requires=containerd.service + +[Service] +ExecStart=/usr/local/bin/kubelet \\ + --config=/var/lib/kubelet/kubelet-config.yaml \\ + --container-runtime=remote \\ + --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock \\ + --image-pull-progress-deadline=2m \\ + --kubeconfig=/var/lib/kubelet/kubeconfig \\ + --network-plugin=cni \\ + --register-node=true \\ + --v=2 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target \ No newline at end of file From 0b212afa41dfaf22905750d5a02b98c49338e8f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Okouya?= Date: Sun, 22 Jul 2018 17:10:27 +0200 Subject: [PATCH 2/7] .gitignore --- .gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b9d9154 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +kubectl +kubectl.kubeconfig +provisioning/mitogen-master +provisioning/pki +readme.txt +ansible-2.5.4 +.DS_Store +.vagrant/ From 0200bd91f67ff6a0d50995a936580009da4ff9f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Okouya?= Date: Sun, 22 Jul 2018 17:12:41 +0200 Subject: [PATCH 3/7] gitignore --- gitignore | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 gitignore diff --git a/gitignore b/gitignore deleted file mode 100644 index 6f179f1..0000000 --- a/gitignore +++ /dev/null @@ -1,7 +0,0 @@ -kubectl -kubectl.kubeconfig -provisioning/mitogen-master -provisioning/pki -readme.txt -ansible-2.5.4 -.DS_Store From d4af7e6f489c2addc34d1148627450fac4c61148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Okouya?= Date: Sun, 22 Jul 2018 17:15:48 +0200 Subject: [PATCH 4/7] .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b9d9154..8dc6bff 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ provisioning/pki readme.txt ansible-2.5.4 .DS_Store -.vagrant/ +.vagrant/ \ No newline at end of file From afb85c243195aa6cfd32de9c577ab4069c9d4d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Okouya?= Date: Sun, 22 Jul 2018 17:58:59 +0200 Subject: [PATCH 5/7] calico.yml.j2 --- .../calico/templates/calico.yml.j2 | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/provisioning/roles/rok.kube-network/calico/templates/calico.yml.j2 b/provisioning/roles/rok.kube-network/calico/templates/calico.yml.j2 index fd64a56..358477b 100644 --- a/provisioning/roles/rok.kube-network/calico/templates/calico.yml.j2 +++ b/provisioning/roles/rok.kube-network/calico/templates/calico.yml.j2 @@ -227,8 +227,8 @@ spec: # Calico policy and networking mode. --- +# description: Calico Felix Configuration apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Felix Configuration kind: CustomResourceDefinition metadata: name: felixconfigurations.crd.projectcalico.org @@ -242,9 +242,8 @@ spec: singular: felixconfiguration --- - +# description: Calico BGP Peers apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico BGP Peers kind: CustomResourceDefinition metadata: name: bgppeers.crd.projectcalico.org @@ -258,9 +257,8 @@ spec: singular: bgppeer --- - +# description: Calico BGP Configuration apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico BGP Configuration kind: CustomResourceDefinition metadata: name: bgpconfigurations.crd.projectcalico.org @@ -274,9 +272,8 @@ spec: singular: bgpconfiguration --- - +# description: Calico IP Pools apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico IP Pools kind: CustomResourceDefinition metadata: name: ippools.crd.projectcalico.org @@ -290,9 +287,8 @@ spec: singular: ippool --- - +# description: Calico HostEndpoints apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico HostEndpoints kind: CustomResourceDefinition metadata: name: hostendpoints.crd.projectcalico.org @@ -306,9 +302,8 @@ spec: singular: hostendpoint --- - +# description: Calico Cluster Information apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Cluster Information kind: CustomResourceDefinition metadata: name: clusterinformations.crd.projectcalico.org @@ -322,9 +317,8 @@ spec: singular: clusterinformation --- - +# description: Calico Global Network Policies apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Global Network Policies kind: CustomResourceDefinition metadata: name: globalnetworkpolicies.crd.projectcalico.org @@ -339,8 +333,8 @@ spec: --- +# description: Calico Global Network Sets apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Global Network Sets kind: CustomResourceDefinition metadata: name: globalnetworksets.crd.projectcalico.org @@ -354,9 +348,8 @@ spec: singular: globalnetworkset --- - +# description: Calico Network Policies apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Network Policies kind: CustomResourceDefinition metadata: name: networkpolicies.crd.projectcalico.org From 47f82c2d38e1a8398be37f90514e14b91b6b9f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Okouya?= Date: Sun, 22 Jul 2018 18:15:37 +0200 Subject: [PATCH 6/7] kubectl.kubeconfig --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 5e6e292..b2dfec7 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -112,6 +112,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| trigger.name = "Cleanup generated ressources" trigger.ignore = [:up, :halt, :resume, :provision, :reload] trigger.info = "Deleting resource folder ./provisioning/pki" - trigger.run = {inline: "rm -rf ./provisioning/pki && rm ./kubectl ./kubeconfig.kubectl"} + trigger.run = {inline: "rm -rf ./provisioning/pki && rm ./kubectl ./kubectl.kubeconfig"} end end \ No newline at end of file From b9ee752c6fe6afeef292988296dbf04830c127c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Okouya?= Date: Sun, 22 Jul 2018 19:11:43 +0200 Subject: [PATCH 7/7] adding conditionnal apps installation based on ansible boolean variable --- clustervars.yml | 6 +++--- provisioning/clusterapps.yml | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/clustervars.yml b/clustervars.yml index 42c0b90..9150d32 100644 --- a/clustervars.yml +++ b/clustervars.yml @@ -54,12 +54,12 @@ ingress: vagrant_enabled: false nodename: node02 -kubeapps: +apps: dashboard: enbaled: false - traefik: + traefik: enaled: true - spinnaker: + spinnaker: enabled: true diff --git a/provisioning/clusterapps.yml b/provisioning/clusterapps.yml index 0e358c6..ecf4039 100644 --- a/provisioning/clusterapps.yml +++ b/provisioning/clusterapps.yml @@ -17,6 +17,7 @@ import_role: name: rok.kube-apps/heapster tasks_from: heapStart + when: apps.dashboard.enabled |default(false) | bool == true - name: start | Deploy Dashboard @@ -24,13 +25,19 @@ import_role: name: rok.kube-apps/dashboard tasks_from: kubeDash + when: apps.dashboard.enabled |default(false) | bool == true - name: Deploy Traefik Ingress Controler tags: ['traefik'] import_role: name: rok.kube-apps/traefik + when: apps.traefik.enabled |default(true) | bool == true - + # - name: Deploy Spinnaker Continuous Delivery Platform + # tags: ['traefik'] + # import_role: + # name: rok.kube-apps/spinnaker + # when: kubeapps.spinnaker.enabled |default(false) | bool == true # spinnaker.apps.roklab.ops # dashboard.apps.roklab.ops