From 57422821980f59f83856f231527b48ae625b54c1 Mon Sep 17 00:00:00 2001
From: jonathantan <359150423@qq.com>
Date: Thu, 26 Oct 2023 12:18:48 +0800
Subject: [PATCH] =?UTF-8?q?K8S=E5=8A=A0=E5=9B=BA=E6=93=8D=E4=BD=9C?=
 =?UTF-8?q?=E8=B0=83=E6=95=B4=E4=BF=AE=E6=94=B9=E6=96=B9=E5=BC=8F=EF=BC=8C?=
 =?UTF-8?q?=E9=81=BF=E5=85=8D=E5=A4=9A=E6=AC=A1=E9=87=8D=E5=90=AF=E6=8E=A7?=
 =?UTF-8?q?=E5=88=B6=E9=9D=A2pod?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

1.kube-proxy修改为只在第一台master操作
2.yaml配置文件修改备份与操作的目录
---
 bcs-ops/k8s/install_k8s | 122 +++++++++++++++++++++++-----------------
 1 file changed, 70 insertions(+), 52 deletions(-)

diff --git a/bcs-ops/k8s/install_k8s b/bcs-ops/k8s/install_k8s
index ddda8c7039..eb5f43be0a 100644
--- a/bcs-ops/k8s/install_k8s
+++ b/bcs-ops/k8s/install_k8s
@@ -42,85 +42,93 @@ if [ -z "${goversion}" ];then
   job_fail "get go version failed, configure etcd failed"
 fi
 
+tmp_dir="/tmp/backup/$(date +%s)"
+mkdir -p ${tmp_dir}
+cp /etc/kubernetes/manifests/* ${tmp_dir}/
+cp /etc/kubernetes/manifests/* ${ROOT_DIR}/
 pod_files=(etcd.yaml kube-apiserver.yaml kube-controller-manager.yaml kube-scheduler.yaml)
 
 for pod_file in ${pod_files[@]};do
   if [[ ${goversion} -le 15 ]] || [[ ${goversion} -ge 12 ]];then
-    if ! grep GODEBUG /etc/kubernetes/manifests/${pod_file};then
-      if [[ $(yq '.spec.containers[0].env' /etc/kubernetes/manifests/${pod_file}) != "null" ]];then
-        env_length=$(yq '.spec.containers[0].env|to_entries|length'  /etc/kubernetes/manifests/${pod_file})
-        yq -i '.spec.containers[0].env['${env_length}']={"name":"GODEBUG", "value":"madvdontneed=1"}' /etc/kubernetes/manifests/${pod_file}
+    if ! grep GODEBUG ${ROOT_DIR}/${pod_file};then
+      if [[ $(yq '.spec.containers[0].env' ${ROOT_DIR}/${pod_file}) != "null" ]];then
+        env_length=$(yq '.spec.containers[0].env|to_entries|length'  ${ROOT_DIR}/${pod_file})
+        yq -i '.spec.containers[0].env['${env_length}']={"name":"GODEBUG", "value":"madvdontneed=1"}' ${ROOT_DIR}/${pod_file}
       else
-        yq -i '.spec.containers[0].env[0]={"name":"GODEBUG", "value":"madvdontneed=1"}' /etc/kubernetes/manifests/${pod_file}
+        yq -i '.spec.containers[0].env[0]={"name":"GODEBUG", "value":"madvdontneed=1"}' ${ROOT_DIR}/${pod_file}
       fi
     fi
   fi
 done
 
-if ! grep -v "^#" /etc/kubernetes/manifests/kube-apiserver.yaml|grep max-mutating-requests-inflight;then
-    yq -i '.spec.containers[0].command += "--max-mutating-requests-inflight=1000"' /etc/kubernetes/manifests/kube-apiserver.yaml
+if ! grep -v "^#" ${ROOT_DIR}/kube-apiserver.yaml|grep max-mutating-requests-inflight;then
+    yq -i '.spec.containers[0].command += "--max-mutating-requests-inflight=1000"' ${ROOT_DIR}/kube-apiserver.yaml
 else
-    if ! grep max-mutating-requests-inflight=1000 /etc/kubernetes/manifests/kube-apiserver.yaml;then
-      element_index=$(yq '.spec.containers[0].command|to_entries'  /etc/kubernetes/manifests/kube-apiserver.yaml|yq '.[]|select (.value|test("max-mutating-requests-inflight")).key')
-      yq -i '.spec.containers[0].command['${element_index}']="--max-mutating-requests-inflight=1000"' /etc/kubernetes/manifests/kube-apiserver.yaml
+    if ! grep max-mutating-requests-inflight=1000 ${ROOT_DIR}/kube-apiserver.yaml;then
+      element_index=$(yq '.spec.containers[0].command|to_entries'  ${ROOT_DIR}/kube-apiserver.yaml|yq '.[]|select (.value|test("max-mutating-requests-inflight")).key')
+      yq -i '.spec.containers[0].command['${element_index}']="--max-mutating-requests-inflight=1000"' ${ROOT_DIR}/kube-apiserver.yaml
     fi
 fi
 
-if ! grep -v "^#" /etc/kubernetes/manifests/kube-apiserver.yaml |grep max-requests-inflight;then
-    yq -i '.spec.containers[0].command += "--max-requests-inflight=3000"' /etc/kubernetes/manifests/kube-apiserver.yaml
+if ! grep -v "^#" ${ROOT_DIR}/kube-apiserver.yaml |grep max-requests-inflight;then
+    yq -i '.spec.containers[0].command += "--max-requests-inflight=3000"' ${ROOT_DIR}/kube-apiserver.yaml
 else
-    if ! grep max-requests-inflight=3000 /etc/kubernetes/manifests/kube-apiserver.yaml ;then
-      element_index=$(yq '.spec.containers[0].command|to_entries'  /etc/kubernetes/manifests/kube-apiserver.yaml|yq '.[]|select (.value|test("max-mutating-requests-inflight")).key')
-      yq -i '.spec.containers[0].command['${element_index}']="--max-requests-inflight=3000"' /etc/kubernetes/manifests/kube-apiserver.yaml
+    if ! grep max-requests-inflight=3000 ${ROOT_DIR}/kube-apiserver.yaml ;then
+      element_index=$(yq '.spec.containers[0].command|to_entries'  ${ROOT_DIR}/kube-apiserver.yaml|yq '.[]|select (.value|test("max-mutating-requests-inflight")).key')
+      yq -i '.spec.containers[0].command['${element_index}']="--max-requests-inflight=3000"' ${ROOT_DIR}/kube-apiserver.yaml
     fi
 fi
 
-if ! grep -v "^#" /etc/kubernetes/manifests/kube-controller-manager.yaml|grep kube-api-qps;then
-    yq -i '.spec.containers[0].command += "--kube-api-qps=300"' /etc/kubernetes/manifests/kube-controller-manager.yaml
+if ! grep -v "^#" ${ROOT_DIR}/kube-controller-manager.yaml|grep kube-api-qps;then
+    yq -i '.spec.containers[0].command += "--kube-api-qps=300"' ${ROOT_DIR}/kube-controller-manager.yaml
 else
-    if ! grep kube-api-qps=300 /etc/kubernetes/manifests/kube-controller-manager.yaml ;then
-      element_index=$(yq '.spec.containers[0].command|to_entries'  /etc/kubernetes/manifests/kube-controller-manager.yaml|yq '.[]|select (.value|test("kube-api-qps")).key')
-      yq -i '.spec.containers[0].command['${element_index}']="--kube-api-qps=300"' /etc/kubernetes/manifests/kube-controller-manager.yaml
+    if ! grep kube-api-qps=300 ${ROOT_DIR}/kube-controller-manager.yaml ;then
+      element_index=$(yq '.spec.containers[0].command|to_entries'  ${ROOT_DIR}/kube-controller-manager.yaml|yq '.[]|select (.value|test("kube-api-qps")).key')
+      yq -i '.spec.containers[0].command['${element_index}']="--kube-api-qps=300"' ${ROOT_DIR}/kube-controller-manager.yaml
     fi
 fi
 
-if ! grep -v "^#" /etc/kubernetes/manifests/kube-controller-manager.yaml |grep kube-api-burst;then
-    yq -i '.spec.containers[0].command += "--kube-api-burst=400"' /etc/kubernetes/manifests/kube-controller-manager.yaml
+if ! grep -v "^#" ${ROOT_DIR}/kube-controller-manager.yaml |grep kube-api-burst;then
+    yq -i '.spec.containers[0].command += "--kube-api-burst=400"' ${ROOT_DIR}/kube-controller-manager.yaml
 else
-    if ! grep kube-api-burst=400 /etc/kubernetes/manifests/kube-controller-manager.yaml;then
-      element_index=$(yq '.spec.containers[0].command|to_entries'  /etc/kubernetes/manifests/kube-controller-manager.yaml|yq '.[]|select (.value|test("kube-api-burst")).key')
-      yq -i '.spec.containers[0].command['${element_index}']="--kube-api-burst=400"' /etc/kubernetes/manifests/kube-controller-manager.yaml
+    if ! grep kube-api-burst=400 ${ROOT_DIR}/kube-controller-manager.yaml;then
+      element_index=$(yq '.spec.containers[0].command|to_entries'  ${ROOT_DIR}/kube-controller-manager.yaml|yq '.[]|select (.value|test("kube-api-burst")).key')
+      yq -i '.spec.containers[0].command['${element_index}']="--kube-api-burst=400"' ${ROOT_DIR}/kube-controller-manager.yaml
     fi
 fi
 
-if ! grep -v "^#" /etc/kubernetes/manifests/kube-controller-manager.yaml|grep terminated-pod-gc-threshold;then
-    yq -i '.spec.containers[0].command += "--terminated-pod-gc-threshold=12500"' /etc/kubernetes/manifests/kube-controller-manager.yaml
+if ! grep -v "^#" ${ROOT_DIR}/kube-controller-manager.yaml|grep terminated-pod-gc-threshold;then
+    yq -i '.spec.containers[0].command += "--terminated-pod-gc-threshold=12500"' ${ROOT_DIR}/kube-controller-manager.yaml
 else
-    if ! grep terminated-pod-gc-threshold=12500 /etc/kubernetes/manifests/kube-controller-manager.yaml;then
-      element_index=$(yq '.spec.containers[0].command|to_entries'  /etc/kubernetes/manifests/kube-controller-manager.yaml|yq '.[]|select (.value|test("terminated-pod-gc-threshold")).key')
-      yq -i '.spec.containers[0].command['${element_index}']="--terminated-pod-gc-threshold=12500"' /etc/kubernetes/manifests/kube-controller-manager.yaml
+    if ! grep terminated-pod-gc-threshold=12500 ${ROOT_DIR}/kube-controller-manager.yaml;then
+      element_index=$(yq '.spec.containers[0].command|to_entries'  ${ROOT_DIR}/kube-controller-manager.yaml|yq '.[]|select (.value|test("terminated-pod-gc-threshold")).key')
+      yq -i '.spec.containers[0].command['${element_index}']="--terminated-pod-gc-threshold=12500"' ${ROOT_DIR}/kube-controller-manager.yaml
     fi
 fi
 
-if ! grep -v "^#" /etc/kubernetes/manifests/kube-scheduler.yaml|grep kube-api-qps;then
-    yq -i '.spec.containers[0].command += "--kube-api-qps=300"' /etc/kubernetes/manifests/kube-scheduler.yaml
+if ! grep -v "^#" ${ROOT_DIR}/kube-scheduler.yaml|grep kube-api-qps;then
+    yq -i '.spec.containers[0].command += "--kube-api-qps=300"' ${ROOT_DIR}/kube-scheduler.yaml
 else
-    if ! grep kube-api-qps=300 /etc/kubernetes/manifests/kube-scheduler.yaml;then
-      element_index=$(yq '.spec.containers[0].command|to_entries'  /etc/kubernetes/manifests/kube-scheduler.yaml|yq '.[]|select (.value|test("kube-api-qps")).key')
-      yq -i '.spec.containers[0].command['${element_index}']="--kube-api-qps=300"' /etc/kubernetes/manifests/kube-scheduler.yaml
+    if ! grep kube-api-qps=300 ${ROOT_DIR}/kube-scheduler.yaml;then
+      element_index=$(yq '.spec.containers[0].command|to_entries'  ${ROOT_DIR}/kube-scheduler.yaml|yq '.[]|select (.value|test("kube-api-qps")).key')
+      yq -i '.spec.containers[0].command['${element_index}']="--kube-api-qps=300"' ${ROOT_DIR}/kube-scheduler.yaml
     fi
 fi
 
-if ! grep -v "^#" /etc/kubernetes/manifests/kube-scheduler.yaml |grep kube-api-burst;then
-    yq -i '.spec.containers[0].command += "--kube-api-burst=400"' /etc/kubernetes/manifests/kube-scheduler.yaml
+if ! grep -v "^#" ${ROOT_DIR}/kube-scheduler.yaml |grep kube-api-burst;then
+    yq -i '.spec.containers[0].command += "--kube-api-burst=400"' ${ROOT_DIR}/kube-scheduler.yaml
 else
-    if ! grep kube-api-burst=400 /etc/kubernetes/manifests/kube-scheduler.yaml;then
-      element_index=$(yq '.spec.containers[0].command|to_entries'  /etc/kubernetes/manifests/kube-scheduler.yaml|yq '.[]|select (.value|test("kube-api-burst")).key')
-      yq -i '.spec.containers[0].command['${element_index}']="--kube-api-burst=400"' /etc/kubernetes/manifests/kube-scheduler.yaml
+    if ! grep kube-api-burst=400 ${ROOT_DIR}/kube-scheduler.yaml;then
+      element_index=$(yq '.spec.containers[0].command|to_entries'  ${ROOT_DIR}/kube-scheduler.yaml|yq '.[]|select (.value|test("kube-api-burst")).key')
+      yq -i '.spec.containers[0].command['${element_index}']="--kube-api-burst=400"' ${ROOT_DIR}/kube-scheduler.yaml
     fi
 fi
 
-sleep 20
+for pod_file in ${pod_files[@]};do
+  cp ${ROOT_DIR}/${pod_file} /etc/kubernetes/manifests/
+done
+
+sleep 30
 pods=(etcd kube-apiserver kube-controller-manager kube-scheduler)
 for pod in ${pods[@]};do
   case "${CRI_TYPE,,}" in
@@ -141,21 +149,31 @@ for pod in ${pods[@]};do
   esac
 done
 
-kubectl get cm -n kube-system kube-proxy -o yaml|yq '.data.["kubeconfig.conf"]' > kubeconfig.conf
-kubectl get cm -n kube-system kube-proxy -o yaml|yq '.data.["config.conf"]'|yq '.ipvs.udpTimeout="10s"' > config.conf
-kubectl get cm -n kube-system kube-proxy  -o yaml > kube-proxy-configmap.bak
-kubectl delete cm kube-proxy -n kube-system 
-kubectl create cm kube-proxy -n kube-system --from-file config.conf --from-file kubeconfig.conf
+if [[ -z ${MASTER_JOIN_CMD:-} ]]; then
+  kubectl get cm -n kube-system kube-proxy -o yaml > ${tmp_dir}/kube-proxy-cm.yaml
+  kubectl get ds -n kube-system kube-proxy -o yaml > ${tmp_dir}/kube-proxy.yaml
 
-if ! kubectl get ds -n kube-system kube-proxy -o yaml|grep madvdontneed;then
-    kubectl patch ds -n kube-system kube-proxy -p '[{"op": "add", "path": "/spec/template/spec/containers/0/env/-", "value":{"name":"GODEBUG", "value":"madvdontneed=1"}}]' --type json
-else
-  if ! kubectl get ds -n kube-system kube-proxy -o yaml|grep madvdontneed=1;then
-    element_index=`kubectl get ds -n kube-system kube-proxy -o json|jq '.spec.template.spec.containers[0].env|to_entries[]|select (.value.name|test("GODEBUG")).key'`
-    kubectl patch ds -n kube-system kube-proxy -p '[{"op": "replace", "path": "/spec/template/spec/containers/0/env/'${element_index}'", "value":{"name":"GODEBUG", "value":"madvdontneed=1"}}]' --type json
+  kubectl get cm -n kube-system kube-proxy -o yaml|yq '.data.["kubeconfig.conf"]' > ${ROOT_DIR}/kubeconfig.conf
+  kubectl get cm -n kube-system kube-proxy -o yaml|yq '.data.["config.conf"]'|yq '.ipvs.udpTimeout="10s"' > ${ROOT_DIR}/config.conf
+  kubectl delete cm kube-proxy -n kube-system 
+  kubectl create cm kube-proxy -n kube-system --from-file config.conf --from-file kubeconfig.conf
+
+  if ! kubectl get ds -n kube-system kube-proxy -o yaml|grep madvdontneed;then
+      kubectl patch ds -n kube-system kube-proxy -p '[{"op": "add", "path": "/spec/template/spec/containers/0/env/-", "value":{"name":"GODEBUG", "value":"madvdontneed=1"}}]' --type json
+  else
+    if ! kubectl get ds -n kube-system kube-proxy -o yaml|grep madvdontneed=1;then
+      element_index=`kubectl get ds -n kube-system kube-proxy -o json|jq '.spec.template.spec.containers[0].env|to_entries[]|select (.value.name|test("GODEBUG")).key'`
+      kubectl patch ds -n kube-system kube-proxy -p '[{"op": "replace", "path": "/spec/template/spec/containers/0/env/'${element_index}'", "value":{"name":"GODEBUG", "value":"madvdontneed=1"}}]' --type json
+    fi
+  fi
+
+  kubectl rollout restart ds -n kube-system kube-proxy
+  if ! kubectl rollout status ds -n kube-system kube-proxy --timeout 60s;then
+    utils::log "FATAL" "Update kube-proxy failed."
   fi
 fi
 
+
 #coredns configuration
 
 utils::log "OK" "K8S configuration done!"
\ No newline at end of file