feat: adding config for crictl

feat: cleanup remaining virtual network interface, container and backup mounts
fix: bap use K8S_CRTL_IP as rs by default。
fix: cleanup node explict specify cri
fix: explicit declaration sock schema
fix: kubelet explict specify cgroupDriver as systemd
fix: controlplane restart kubelet and wait
style：shell format

bcs-ops/clean_master.sh

@@ -17,7 +17,7 @@ ROOT_DIR=${SELF_DIR}
 readonly SELF_DIR ROOT_DIR
 
 # only 1.2[0-1] to run
-kubeadm reset phase update-cluster-status || true
-kubeadm reset phase remove-etcd-member
+kubeadm reset phase update-cluster-status --v=5 || true
+kubeadm reset phase remove-etcd-member --v=5
 
 "${ROOT_DIR}"/clean_node.sh

bcs-ops/clean_node.sh

@@ -18,20 +18,6 @@ ROOT_DIR=${SELF_DIR}
 
 readonly BAK_DIR SELF_DIR ROOT_DIR
 
-clean_container() {
-  case "${CRI_TYPE,,}" in
-    "containerd")
-      ctr -n k8s.io t ls | grep -qv PID && ctr -n k8s.io t rm -f "$(ctr -n k8s.io t ls -q)"
-      ctr -n k8s.io c ls | grep -qv CONTAINER && ctr -n k8s.io c rm "$(ctr -n k8s.io c ls -q)"
-      systemctl disable --now containerd
-      ;;
-    "docker")
-      docker ps | grep -qv NAME && docker rm -f "$(docker ps -aq)"
-      systemctl disable --now docker
-      ;;
-  esac
-}
-
 safe_source() {
   local source_file=$1
   if [[ -f ${source_file} ]]; then
@@ -48,8 +34,30 @@ for file in "${source_files[@]}"; do
   safe_source "$file"
 done
 
+clean_container() {
+  crictl ps -aq | xargs -r crictl rm -f
+}
+
+clean_cni() {
+  case ${K8S_CNI} in
+    "flannel")
+      ip l | awk '/flannel/{eth=$2;gsub(":","",eth);print eth}' | xargs -r -n 1 ip l d
+      ;;
+    *)
+      return 0
+      ;;
+  esac
+  rm -rf /etc/cni/net.d/*
+}
+
+clean_vni() {
+  ip l | awk '/cni0|kube-ipvs0/{eth=$2;gsub(":","",eth);print eth}' | xargs -r -n 1 ip l d
+  ip l | awk '/veth/{eth=$2;split(eth,a,"@");print a[1]}' | xargs -r -n 1 ip l d
+}
+
+kubeadm reset phase cleanup-node \
+  --cri-socket "$(crictl config --get runtime-endpoint)" --v=5
 systemctl disable --now kubelet
-kubeadm reset phase cleanup-node
 clean_container
 
 ip l d cni0 || utils::log "WARN" "link cni0 does not exist"
@@ -79,3 +87,7 @@ utils::log "OK" "Back Files >>> Done"
 
 "${ROOT_DIR}"/system/config_iptables.sh clean \
   && utils::log "OK" "Clean k8s-components iptables rules"
+
+df -h | awk '/backup.*kubelet/{print $NF}' | xargs -r umount
+clean_cni
+clean_vni

bcs-ops/env/offline-manifest.yaml

@@ -3,7 +3,7 @@ bcs-ops:
     bin-tools:
       k8s: "1.20.15"
       cni-plugins: "1.3.0"
-      crictl: "1.26.0"
+      crictl: "1.24.2"
       containerd: "1.6.21"
       runc: "1.1.8"
       docker: "19.03.9"
@@ -42,7 +42,7 @@ bcs-ops:
     bin-tools:
       k8s: "1.23.17"
       cni-plugins: "1.3.0"
-      crictl: "1.26.0"
+      crictl: "1.24.2"
       containerd: "1.6.21"
       runc: "1.1.8"
       docker: "19.03.9"
@@ -80,7 +80,7 @@ bcs-ops:
     bin-tools:
       k8s: "1.24.15"
       cni-plugins: "1.3.0"
-      crictl: "1.26.0"
+      crictl: "1.24.2"
       containerd: "1.6.21"
       runc: "1.1.8"
       jq: "1.6"

bcs-ops/functions/k8s.sh

@@ -39,9 +39,9 @@ k8s::safe_add_helmrepo() {
     utils::log "INFO" "remove old helm repo: $repo_name"
     helm repo remove "$repo_name"
   fi
-  if ! helm repo add "$repo_name" "$repo_url";then
-	utils::log "ERROR" "can't add helm repo $repo_name $repo_url"
-	return 1
+  if ! helm repo add "$repo_name" "$repo_url"; then
+    utils::log "ERROR" "can't add helm repo $repo_name $repo_url"
+    return 1
   fi
   helm repo list
   if ! helm repo update; then
@@ -98,3 +98,18 @@ k8s::restart_kubelet() {
   utils::log "ERROR" "restart kubelet service failed"
   return 1
 }
+
+k8s::check_master() {
+  local timeout=5
+  while ((timeout > 0)); do
+    if ! kubectl cluster-info; then
+      utils::log "WARN" "timeout=$timeout, \
+controlplane has not been started yet, waiting"
+      crictl ps
+    else
+      return 0
+    fi
+	sleep 30
+  done
+  return 1
+}

bcs-ops/install_master.sh

@@ -100,6 +100,8 @@ else
   "${ROOT_DIR}"/system/config_envfile.sh -c clean
   K8S_CTRL_IP=$LAN_IP "${ROOT_DIR}"/system/config_envfile.sh -c init
   k8s::restart_kubelet
+  sleep 30
+  k8s::check_master
   if [[ ${ENABLE_APISERVER_HA} == "true" ]]; then
     [[ -z ${VIP} ]] && utils::log "ERROR" "apiserver HA is enabled but VIP is not set"
     if [[ ${APISERVER_HA_MODE} == "kube-vip" ]]; then

bcs-ops/install_node.sh

@@ -95,7 +95,7 @@ init_bap_rule() {
   esac
 
   [[ -z "${VIP}" ]] && utils::log "ERROR" "apiserver HA is enabled but VIP is not set"
-  "${PROXY_TOOL_PATH}"/bcs-apiserver-proxy-tools -cmd init -vs "${VIP}":"${VS_PORT}" -rs "${LAN_IP}":6443 \
+  "${PROXY_TOOL_PATH}"/bcs-apiserver-proxy-tools -cmd init -vs "${VIP}":"${VS_PORT}" -rs "${K8S_CTRL_IP}":6443 \
     -scheduler "${LVS_SCHEDULER}" -toolPath "${PROXY_TOOL_PATH}"/bcs-apiserver-proxy-tools
   "${ROOT_DIR}"/system/config_bcs_dns -u "${VIP}" k8s-api.bcs.local
   k8s::restart_kubelet

bcs-ops/install_op.sh

@@ -35,7 +35,7 @@ for file in "${source_files[@]}"; do
   safe_source "$file"
 done
 
-"${ROOT_DIR}"/k8s/operate_completion kubeadm kubectl helm ctr yq
+"${ROOT_DIR}"/k8s/operate_completion kubeadm kubectl helm ctr yq crictl
 
 if [[ -n "${BKREPO_URL:-}" ]]; then
   if command -v helm &>/dev/null; then

bcs-ops/k8s/install_containerd

@@ -225,7 +225,7 @@ main() {
   fi
 
   if [[ -n ${BCS_OFFLINE:-} ]]; then
-    find "${ROOT_DIR}"/version-"${VERSION}"/images -name '*.tar' -type f -print0 \
+    find "${ROOT_DIR}"/version-"${K8S_VER}"/images -name '*.tar' -type f -print0 \
       | xargs -0 -I {} ctr -n k8s.io image import {}
   fi
 

bcs-ops/k8s/install_docker

@@ -207,7 +207,7 @@ main() {
 
   # bcs_offline load image
   if [[ -n ${BCS_OFFLINE:-} ]]; then
-    find "${ROOT_DIR}"/version-"${VERSION}"/images -name '*.tar' -type f -print0 \
+    find "${ROOT_DIR}"/version-"${K8S_VER}"/images -name '*.tar' -type f -print0 \
       | xargs -0 -I {} docker load -i {}
   fi
 

bcs-ops/k8s/install_k8s_tools

@@ -69,7 +69,13 @@ _yum_k8s() {
     | awk '/'"$pkg_pat"'/{last=$2} END{sub(/.*:/,"",last);print last}')"
   [[ -n $pkg_ver ]] \
     || utils::log "ERROR" "${K8S_VER:-} not found amongst yum list results"
-  yum install -y "kubeadm-${pkg_ver}" "kubelet-${pkg_ver}" "kubectl-${pkg_ver}"
+  local crictl_pat crictl_ver
+  crictl_pat=$(awk '/crictl:/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
+
+  crictl_ver=$(yum list cri-tools --showduplicates \
+    | awk '/'"${crictl_pat}"'/{last=$2} END{sub(/.*:/,"",last);print last}')
+
+  yum install -y "kubeadm-${pkg_ver}" "kubelet-${pkg_ver}" "kubectl-${pkg_ver}" "cri-tools-${crictl_ver}"
 }
 
 _curl_k8s() {
@@ -78,7 +84,9 @@ _curl_k8s() {
   mkdir -p "$bin_path"
 
   name="k8s"
-  ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
+  ver=$(awk \
+    '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' \
+    "${ROOT_DIR}"/env/offline-manifest.yaml)
   file="${name}-${ver}.tgz"
   url=${REPO_URL:-}/${file}
   if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
@@ -88,7 +96,9 @@ _curl_k8s() {
   fi
 
   name="crictl"
-  ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
+  ver=$(awk \
+    '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' \
+    "${ROOT_DIR}"/env/offline-manifest.yaml)
   file="${name}-${ver}.tgz"
   url="${REPO_URL}/${file}"
   if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
@@ -98,7 +108,9 @@ _curl_k8s() {
   fi
 
   name="cni-plugins"
-  ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
+  ver=$(awk \
+    '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' \
+    "${ROOT_DIR}"/env/offline-manifest.yaml)
   file="${name}-${ver}.tgz"
   url="${REPO_URL}/${file}"
   if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
@@ -141,6 +153,28 @@ _offline_k8s() {
   fi
 }
 
+config_crictl() {
+  local sock config_file
+  config_file="/etc/crictl.yaml"
+  [[ -f $config_file ]] || touch $config_file
+  case "${CRI_TYPE,,}" in
+    "docker")
+      sock="unix:///var/run/dockershim.sock"
+      ;;
+    "containerd")
+      sock="unix:///run/containerd/containerd.sock"
+      ;;
+    *)
+      # ToDo: Unified standard error code
+      export ERR_CODE=1
+      utils::log FATAL "unkown CRI_TYPE:$CRI_TYPE"
+      ;;
+  esac
+  crictl config runtime-endpoint $sock
+  crictl config image-endpoint $sock
+  "${ROOT_DIR}"/k8s/operate_completion crictl
+}
+
 main() {
   local source_files
   source_files=("${ROOT_DIR}/functions/utils.sh" "${ROOT_DIR}/env/bcs.env")
@@ -164,6 +198,8 @@ main() {
     esac
   fi
 
+  config_crictl
+
   utils::log "INFO" "check kubeadm status"
   if kubeadm version -o short; then
     utils::log "OK" "kubeadm install success"

bcs-ops/k8s/operate_completion

@@ -22,7 +22,7 @@ ROOT_DIR="${SELF_DIR}/.."
 
 RC_FILE="/etc/bash_completion.d/bcs.bash"
 
-PROJECTS=(kubeadm kubectl helm ctr yq)
+PROJECTS=(kubeadm kubectl crictl helm ctr yq)
 readonly SELF_DIR ROOT_DIR RC_FILE PROJECTS
 
 usage_and_exit() {
@@ -140,6 +140,18 @@ source <(kubeadm completion bash)
 EOF
 }
 
+completion_crictl() {
+  check_completion
+  sed -ri \
+    '/bcs config begin for crictl/,/bcs config end for crictl/d' ${RC_FILE}
+  cat >>"$RC_FILE" <<'EOF'
+# bcs config begin for crictl
+# crictl 命令补全
+source <(crictl completion bash)
+# bcs config end for crictl
+EOF
+}
+
 safe_source() {
   local source_file=$1
   if [[ -f ${source_file} ]]; then