Skip to content

Commit

Permalink
K8S加固操作调整修改方式，避免多次重启控制面pod
Browse files Browse the repository at this point in the history 
1.kube-proxy修改为只在第一台master操作
2.yaml配置文件修改备份与操作的目录
  • Loading branch information
@denglouping
denglouping committed Oct 26, 2023
1 parent 9abb5af commit 5742282
Showing 1 changed file with 70 additions and 52 deletions.
122 changes: 70 additions & 52 deletions bcs-ops/k8s/install_k8s
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,85 +42,93 @@ if [ -z "${goversion}" ];then
job_fail "get go version failed, configure etcd failed"
fi

tmp_dir="/tmp/backup/$(date +%s)"
mkdir -p ${tmp_dir}
cp /etc/kubernetes/manifests/* ${tmp_dir}/
cp /etc/kubernetes/manifests/* ${ROOT_DIR}/
pod_files=(etcd.yaml kube-apiserver.yaml kube-controller-manager.yaml kube-scheduler.yaml)

for pod_file in ${pod_files[@]};do
if [[ ${goversion} -le 15 ]] || [[ ${goversion} -ge 12 ]];then
if ! grep GODEBUG /etc/kubernetes/manifests/${pod_file};then
if [[ $(yq '.spec.containers[0].env' /etc/kubernetes/manifests/${pod_file}) != "null" ]];then
env_length=$(yq '.spec.containers[0].env|to_entries|length' /etc/kubernetes/manifests/${pod_file})
yq -i '.spec.containers[0].env['${env_length}']={"name":"GODEBUG", "value":"madvdontneed=1"}' /etc/kubernetes/manifests/${pod_file}
if ! grep GODEBUG ${ROOT_DIR}/${pod_file};then
if [[ $(yq '.spec.containers[0].env' ${ROOT_DIR}/${pod_file}) != "null" ]];then
env_length=$(yq '.spec.containers[0].env|to_entries|length' ${ROOT_DIR}/${pod_file})
yq -i '.spec.containers[0].env['${env_length}']={"name":"GODEBUG", "value":"madvdontneed=1"}' ${ROOT_DIR}/${pod_file}
else
yq -i '.spec.containers[0].env[0]={"name":"GODEBUG", "value":"madvdontneed=1"}' /etc/kubernetes/manifests/${pod_file}
yq -i '.spec.containers[0].env[0]={"name":"GODEBUG", "value":"madvdontneed=1"}' ${ROOT_DIR}/${pod_file}
fi
fi
fi
done

if ! grep -v "^#" /etc/kubernetes/manifests/kube-apiserver.yaml|grep max-mutating-requests-inflight;then
yq -i '.spec.containers[0].command += "--max-mutating-requests-inflight=1000"' /etc/kubernetes/manifests/kube-apiserver.yaml
if ! grep -v "^#" ${ROOT_DIR}/kube-apiserver.yaml|grep max-mutating-requests-inflight;then
yq -i '.spec.containers[0].command += "--max-mutating-requests-inflight=1000"' ${ROOT_DIR}/kube-apiserver.yaml
else
if ! grep max-mutating-requests-inflight=1000 /etc/kubernetes/manifests/kube-apiserver.yaml;then
element_index=$(yq '.spec.containers[0].command|to_entries' /etc/kubernetes/manifests/kube-apiserver.yaml|yq '.[]|select (.value|test("max-mutating-requests-inflight")).key')
yq -i '.spec.containers[0].command['${element_index}']="--max-mutating-requests-inflight=1000"' /etc/kubernetes/manifests/kube-apiserver.yaml
if ! grep max-mutating-requests-inflight=1000 ${ROOT_DIR}/kube-apiserver.yaml;then
element_index=$(yq '.spec.containers[0].command|to_entries' ${ROOT_DIR}/kube-apiserver.yaml|yq '.[]|select (.value|test("max-mutating-requests-inflight")).key')
yq -i '.spec.containers[0].command['${element_index}']="--max-mutating-requests-inflight=1000"' ${ROOT_DIR}/kube-apiserver.yaml
fi
fi

if ! grep -v "^#" /etc/kubernetes/manifests/kube-apiserver.yaml |grep max-requests-inflight;then
yq -i '.spec.containers[0].command += "--max-requests-inflight=3000"' /etc/kubernetes/manifests/kube-apiserver.yaml
if ! grep -v "^#" ${ROOT_DIR}/kube-apiserver.yaml |grep max-requests-inflight;then
yq -i '.spec.containers[0].command += "--max-requests-inflight=3000"' ${ROOT_DIR}/kube-apiserver.yaml
else
if ! grep max-requests-inflight=3000 /etc/kubernetes/manifests/kube-apiserver.yaml ;then
element_index=$(yq '.spec.containers[0].command|to_entries' /etc/kubernetes/manifests/kube-apiserver.yaml|yq '.[]|select (.value|test("max-mutating-requests-inflight")).key')
yq -i '.spec.containers[0].command['${element_index}']="--max-requests-inflight=3000"' /etc/kubernetes/manifests/kube-apiserver.yaml
if ! grep max-requests-inflight=3000 ${ROOT_DIR}/kube-apiserver.yaml ;then
element_index=$(yq '.spec.containers[0].command|to_entries' ${ROOT_DIR}/kube-apiserver.yaml|yq '.[]|select (.value|test("max-mutating-requests-inflight")).key')
yq -i '.spec.containers[0].command['${element_index}']="--max-requests-inflight=3000"' ${ROOT_DIR}/kube-apiserver.yaml
fi
fi

if ! grep -v "^#" /etc/kubernetes/manifests/kube-controller-manager.yaml|grep kube-api-qps;then
yq -i '.spec.containers[0].command += "--kube-api-qps=300"' /etc/kubernetes/manifests/kube-controller-manager.yaml
if ! grep -v "^#" ${ROOT_DIR}/kube-controller-manager.yaml|grep kube-api-qps;then
yq -i '.spec.containers[0].command += "--kube-api-qps=300"' ${ROOT_DIR}/kube-controller-manager.yaml
else
if ! grep kube-api-qps=300 /etc/kubernetes/manifests/kube-controller-manager.yaml ;then
element_index=$(yq '.spec.containers[0].command|to_entries' /etc/kubernetes/manifests/kube-controller-manager.yaml|yq '.[]|select (.value|test("kube-api-qps")).key')
yq -i '.spec.containers[0].command['${element_index}']="--kube-api-qps=300"' /etc/kubernetes/manifests/kube-controller-manager.yaml
if ! grep kube-api-qps=300 ${ROOT_DIR}/kube-controller-manager.yaml ;then
element_index=$(yq '.spec.containers[0].command|to_entries' ${ROOT_DIR}/kube-controller-manager.yaml|yq '.[]|select (.value|test("kube-api-qps")).key')
yq -i '.spec.containers[0].command['${element_index}']="--kube-api-qps=300"' ${ROOT_DIR}/kube-controller-manager.yaml
fi
fi

if ! grep -v "^#" /etc/kubernetes/manifests/kube-controller-manager.yaml |grep kube-api-burst;then
yq -i '.spec.containers[0].command += "--kube-api-burst=400"' /etc/kubernetes/manifests/kube-controller-manager.yaml
if ! grep -v "^#" ${ROOT_DIR}/kube-controller-manager.yaml |grep kube-api-burst;then
yq -i '.spec.containers[0].command += "--kube-api-burst=400"' ${ROOT_DIR}/kube-controller-manager.yaml
else
if ! grep kube-api-burst=400 /etc/kubernetes/manifests/kube-controller-manager.yaml;then
element_index=$(yq '.spec.containers[0].command|to_entries' /etc/kubernetes/manifests/kube-controller-manager.yaml|yq '.[]|select (.value|test("kube-api-burst")).key')
yq -i '.spec.containers[0].command['${element_index}']="--kube-api-burst=400"' /etc/kubernetes/manifests/kube-controller-manager.yaml
if ! grep kube-api-burst=400 ${ROOT_DIR}/kube-controller-manager.yaml;then
element_index=$(yq '.spec.containers[0].command|to_entries' ${ROOT_DIR}/kube-controller-manager.yaml|yq '.[]|select (.value|test("kube-api-burst")).key')
yq -i '.spec.containers[0].command['${element_index}']="--kube-api-burst=400"' ${ROOT_DIR}/kube-controller-manager.yaml
fi
fi

if ! grep -v "^#" /etc/kubernetes/manifests/kube-controller-manager.yaml|grep terminated-pod-gc-threshold;then
yq -i '.spec.containers[0].command += "--terminated-pod-gc-threshold=12500"' /etc/kubernetes/manifests/kube-controller-manager.yaml
if ! grep -v "^#" ${ROOT_DIR}/kube-controller-manager.yaml|grep terminated-pod-gc-threshold;then
yq -i '.spec.containers[0].command += "--terminated-pod-gc-threshold=12500"' ${ROOT_DIR}/kube-controller-manager.yaml
else
if ! grep terminated-pod-gc-threshold=12500 /etc/kubernetes/manifests/kube-controller-manager.yaml;then
element_index=$(yq '.spec.containers[0].command|to_entries' /etc/kubernetes/manifests/kube-controller-manager.yaml|yq '.[]|select (.value|test("terminated-pod-gc-threshold")).key')
yq -i '.spec.containers[0].command['${element_index}']="--terminated-pod-gc-threshold=12500"' /etc/kubernetes/manifests/kube-controller-manager.yaml
if ! grep terminated-pod-gc-threshold=12500 ${ROOT_DIR}/kube-controller-manager.yaml;then
element_index=$(yq '.spec.containers[0].command|to_entries' ${ROOT_DIR}/kube-controller-manager.yaml|yq '.[]|select (.value|test("terminated-pod-gc-threshold")).key')
yq -i '.spec.containers[0].command['${element_index}']="--terminated-pod-gc-threshold=12500"' ${ROOT_DIR}/kube-controller-manager.yaml
fi
fi

if ! grep -v "^#" /etc/kubernetes/manifests/kube-scheduler.yaml|grep kube-api-qps;then
yq -i '.spec.containers[0].command += "--kube-api-qps=300"' /etc/kubernetes/manifests/kube-scheduler.yaml
if ! grep -v "^#" ${ROOT_DIR}/kube-scheduler.yaml|grep kube-api-qps;then
yq -i '.spec.containers[0].command += "--kube-api-qps=300"' ${ROOT_DIR}/kube-scheduler.yaml
else
if ! grep kube-api-qps=300 /etc/kubernetes/manifests/kube-scheduler.yaml;then
element_index=$(yq '.spec.containers[0].command|to_entries' /etc/kubernetes/manifests/kube-scheduler.yaml|yq '.[]|select (.value|test("kube-api-qps")).key')
yq -i '.spec.containers[0].command['${element_index}']="--kube-api-qps=300"' /etc/kubernetes/manifests/kube-scheduler.yaml
if ! grep kube-api-qps=300 ${ROOT_DIR}/kube-scheduler.yaml;then
element_index=$(yq '.spec.containers[0].command|to_entries' ${ROOT_DIR}/kube-scheduler.yaml|yq '.[]|select (.value|test("kube-api-qps")).key')
yq -i '.spec.containers[0].command['${element_index}']="--kube-api-qps=300"' ${ROOT_DIR}/kube-scheduler.yaml
fi
fi

if ! grep -v "^#" /etc/kubernetes/manifests/kube-scheduler.yaml |grep kube-api-burst;then
yq -i '.spec.containers[0].command += "--kube-api-burst=400"' /etc/kubernetes/manifests/kube-scheduler.yaml
if ! grep -v "^#" ${ROOT_DIR}/kube-scheduler.yaml |grep kube-api-burst;then
yq -i '.spec.containers[0].command += "--kube-api-burst=400"' ${ROOT_DIR}/kube-scheduler.yaml
else
if ! grep kube-api-burst=400 /etc/kubernetes/manifests/kube-scheduler.yaml;then
element_index=$(yq '.spec.containers[0].command|to_entries' /etc/kubernetes/manifests/kube-scheduler.yaml|yq '.[]|select (.value|test("kube-api-burst")).key')
yq -i '.spec.containers[0].command['${element_index}']="--kube-api-burst=400"' /etc/kubernetes/manifests/kube-scheduler.yaml
if ! grep kube-api-burst=400 ${ROOT_DIR}/kube-scheduler.yaml;then
element_index=$(yq '.spec.containers[0].command|to_entries' ${ROOT_DIR}/kube-scheduler.yaml|yq '.[]|select (.value|test("kube-api-burst")).key')
yq -i '.spec.containers[0].command['${element_index}']="--kube-api-burst=400"' ${ROOT_DIR}/kube-scheduler.yaml
fi
fi

sleep 20
for pod_file in ${pod_files[@]};do
cp ${ROOT_DIR}/${pod_file} /etc/kubernetes/manifests/
done

sleep 30
pods=(etcd kube-apiserver kube-controller-manager kube-scheduler)
for pod in ${pods[@]};do
case "${CRI_TYPE,,}" in
Expand All @@ -141,21 +149,31 @@ for pod in ${pods[@]};do
esac
done

kubectl get cm -n kube-system kube-proxy -o yaml|yq '.data.["kubeconfig.conf"]' > kubeconfig.conf
kubectl get cm -n kube-system kube-proxy -o yaml|yq '.data.["config.conf"]'|yq '.ipvs.udpTimeout="10s"' > config.conf
kubectl get cm -n kube-system kube-proxy -o yaml > kube-proxy-configmap.bak
kubectl delete cm kube-proxy -n kube-system
kubectl create cm kube-proxy -n kube-system --from-file config.conf --from-file kubeconfig.conf
if [[ -z ${MASTER_JOIN_CMD:-} ]]; then
kubectl get cm -n kube-system kube-proxy -o yaml > ${tmp_dir}/kube-proxy-cm.yaml
kubectl get ds -n kube-system kube-proxy -o yaml > ${tmp_dir}/kube-proxy.yaml

if ! kubectl get ds -n kube-system kube-proxy -o yaml|grep madvdontneed;then
kubectl patch ds -n kube-system kube-proxy -p '[{"op": "add", "path": "/spec/template/spec/containers/0/env/-", "value":{"name":"GODEBUG", "value":"madvdontneed=1"}}]' --type json
else
if ! kubectl get ds -n kube-system kube-proxy -o yaml|grep madvdontneed=1;then
element_index=`kubectl get ds -n kube-system kube-proxy -o json|jq '.spec.template.spec.containers[0].env|to_entries[]|select (.value.name|test("GODEBUG")).key'`
kubectl patch ds -n kube-system kube-proxy -p '[{"op": "replace", "path": "/spec/template/spec/containers/0/env/'${element_index}'", "value":{"name":"GODEBUG", "value":"madvdontneed=1"}}]' --type json
kubectl get cm -n kube-system kube-proxy -o yaml|yq '.data.["kubeconfig.conf"]' > ${ROOT_DIR}/kubeconfig.conf
kubectl get cm -n kube-system kube-proxy -o yaml|yq '.data.["config.conf"]'|yq '.ipvs.udpTimeout="10s"' > ${ROOT_DIR}/config.conf
kubectl delete cm kube-proxy -n kube-system
kubectl create cm kube-proxy -n kube-system --from-file config.conf --from-file kubeconfig.conf

if ! kubectl get ds -n kube-system kube-proxy -o yaml|grep madvdontneed;then
kubectl patch ds -n kube-system kube-proxy -p '[{"op": "add", "path": "/spec/template/spec/containers/0/env/-", "value":{"name":"GODEBUG", "value":"madvdontneed=1"}}]' --type json
else
if ! kubectl get ds -n kube-system kube-proxy -o yaml|grep madvdontneed=1;then
element_index=`kubectl get ds -n kube-system kube-proxy -o json|jq '.spec.template.spec.containers[0].env|to_entries[]|select (.value.name|test("GODEBUG")).key'`
kubectl patch ds -n kube-system kube-proxy -p '[{"op": "replace", "path": "/spec/template/spec/containers/0/env/'${element_index}'", "value":{"name":"GODEBUG", "value":"madvdontneed=1"}}]' --type json
fi
fi

kubectl rollout restart ds -n kube-system kube-proxy
if ! kubectl rollout status ds -n kube-system kube-proxy --timeout 60s;then
utils::log "FATAL" "Update kube-proxy failed."
fi
fi


#coredns configuration

utils::log "OK" "K8S configuration done!"

0 comments on commit 5742282

Please sign in to comment.