From 081e44f1eabbc7405509adcbea476c9e6e8d9aa1 Mon Sep 17 00:00:00 2001
From: denglouping <359150423@qq.com>
Date: Fri, 1 Mar 2024 16:39:52 +0800
Subject: [PATCH] =?UTF-8?q?=E9=9B=86=E7=BE=A4apiserver=E8=AF=81=E4=B9=A6?=
 =?UTF-8?q?=E6=94=AF=E6=8C=81=E7=9B=B4=E6=8E=A5=E9=85=8D=E7=BD=AEIP?=
 =?UTF-8?q?=E5=92=8C=E5=9F=9F=E5=90=8D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 bcs-ops/k8s/render_kubeadm       | 17 ++++++++++++++++
 bcs-ops/readme.md                | 34 ++++++++++++++++----------------
 bcs-ops/system/config_envfile.sh |  8 ++++++++
 3 files changed, 42 insertions(+), 17 deletions(-)

diff --git a/bcs-ops/k8s/render_kubeadm b/bcs-ops/k8s/render_kubeadm
index e22f56b3e1..4654874cda 100755
--- a/bcs-ops/k8s/render_kubeadm
+++ b/bcs-ops/k8s/render_kubeadm
@@ -53,6 +53,23 @@ render_cluster() {
   cat >"${config_file}" <<EOF
 apiVersion: kubeadm.k8s.io/$kubeadm_tag
 apiServer:
+$(
+    if [[ ${ENABLE_APISERVER_HA} == "true" ]] && [[ ${APISERVER_HA_MODE} == "external" ]];then
+    cat <<EXTERNAL_EOF
+  certSANs:
+EXTERNAL_EOF
+      if [[ -n "${VIP}" ]];then
+    cat <<EXTERNAL_EOF
+    - "${VIP}"
+EXTERNAL_EOF
+      fi
+      if [[ -n "${APISERVER_HOST}" ]];then
+cat <<EXTERNAL_EOF
+    - "${APISERVER_HOST}"
+EXTERNAL_EOF
+      fi
+    fi
+)
   extraArgs:
     authorization-mode: Node,RBAC
 $(
diff --git a/bcs-ops/readme.md b/bcs-ops/readme.md
index e84d94b393..7ff3336f20 100644
--- a/bcs-ops/readme.md
+++ b/bcs-ops/readme.md
@@ -79,7 +79,7 @@ set +x
 
 1. 通过`set -a 命令`配置环境变量，环境变量配置见[`环境变量`](#环境变量)
 2. `./bcs-ops -r bcsenv` 在第一台主机（后称中控机）上渲染配置文件 `env/bcs.env`
-3. 在中控机上启动集群控制平面：`./bcs-ops --instal master`，集群启动成功后会显示加入集群的指令
+3. 在中控机上启动集群控制平面：`./bcs-ops --install master`，集群启动成功后会显示加入集群的指令
 4. 集群加入指令有效期为 1 小时，中控机执行 `./bcs-ops --render joincmd` 可再次渲染生成加入集群的指令，渲染结果如下所示
 
    ```plaintext
@@ -196,22 +196,22 @@ set +x
 
 #### apiserver ha 环境变量
 
-| 环境变量                  | 默认值                      | 说明                                                                                                                                                                                                                         |
-| ------------------------- | --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `ENABLE_APISERVER_HA`     | `false`                     | apiserver ha 模式，默认关闭。                                                                                                                                                                                                |
-| `APISERVER_HA_MODE`       | `bcs-apiserver-proxy`       | 模式选择，支持 [bcs-apiserver-proxy](https://github.com/TencentBlueKing/bk-bcs/blob/625be3183d99ee3500123016a6dea99d78165565/docs/features/bcs-apiserver-proxy/bcs-apiserver-proxy.md#L1)`​[kube-vip](https://kube-vip.io/)` |
-| `VIP`                     |                             | VIP 地址，可配置与集群内不冲突的 ip 地址                                                                                                                                                                                     |
-| `VS_PORT`                 | `6443`                      | bap 代理端口                                                                                                                                                                                                                 |
-| `APISERVER_PROXY_VERSION` | `v1.29.0-alpha.130-tencent` | bap 镜像版本                                                                                                                                                                                                                 |
-| `PROXY_TOOL_PATH`         | `/usr/bin`                  | bap 工具安装目录                                                                                                                                                                                                             |
-| `PERSIST_DIR`             | `/root/.bcs`                | bap 持久化目录                                                                                                                                                                                                               |
-| `LVS_SCHEDULER`           | `rr`                        | bap 负载均衡策略                                                                                                                                                                                                             |
-| `MANAGER_INTERVAL`        | `10`                        | bap 监听时间                                                                                                                                                                                                                 |
-| `DEBUG_MODE`              | `true`                      | bap DEBUG 模式 默认开启                                                                                                                                                                                                      |
-| `LOG_LEVEL`               | `3`                         | bap 日志等级                                                                                                                                                                                                                 |
-| `KUBE_VIP_VERSION`        | `v0.5.12`                   | kube-vip 镜像版本                                                                                                                                                                                                            |
-| `BIND_INTERFACE`          | `""`                        | kube-vip 绑定网卡名                                                                                                                                                                                                          |
-| `VIP_CIDR`                | `32`                        | VIP CIDR 掩码长度                                                                                                                                                                                                            |
+| 环境变量                  | 默认值                      | 说明                                                                                                                                                                                                                             |
+| ------------------------- | --------------------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `ENABLE_APISERVER_HA`     | `false`                     | apiserver ha 模式，默认关闭。                                                                                                                                                                                                          |
+| `APISERVER_HA_MODE`       | `bcs-apiserver-proxy`       | 模式选择，支持 [bcs-apiserver-proxy](https://github.com/TencentBlueKing/bk-bcs/blob/625be3183d99ee3500123016a6dea99d78165565/docs/features/bcs-apiserver-proxy/bcs-apiserver-proxy.md#L1), [kube-vip](https://kube-vip.io/), external |
+| `VIP`                     |                             | VIP 地址，可配置与集群内不冲突的 ip 地址                                                                                                                                                                                                       |
+| `VS_PORT`                 | `6443`                      | bap 代理端口                                                                                                                                                                                                                       |
+| `APISERVER_PROXY_VERSION` | `v1.29.0-alpha.130-tencent` | bap 镜像版本                                                                                                                                                                                                                       |
+| `PROXY_TOOL_PATH`         | `/usr/bin`                  | bap 工具安装目录                                                                                                                                                                                                                     |
+| `PERSIST_DIR`             | `/root/.bcs`                | bap 持久化目录                                                                                                                                                                                                                      |
+| `LVS_SCHEDULER`           | `rr`                        | bap 负载均衡策略                                                                                                                                                                                                                     |
+| `MANAGER_INTERVAL`        | `10`                        | bap 监听时间                                                                                                                                                                                                                       |
+| `DEBUG_MODE`              | `true`                      | bap DEBUG 模式 默认开启                                                                                                                                                                                                              |
+| `LOG_LEVEL`               | `3`                         | bap 日志等级                                                                                                                                                                                                                       |
+| `KUBE_VIP_VERSION`        | `v0.5.12`                   | kube-vip 镜像版本                                                                                                                                                                                                                  |
+| `BIND_INTERFACE`          | `""`                        | kube-vip 绑定网卡名                                                                                                                                                                                                                 |
+| `VIP_CIDR`                | `32`                        | VIP CIDR 掩码长度                                                                                                                                                                                                                  |
 
 ### 示例
 
diff --git a/bcs-ops/system/config_envfile.sh b/bcs-ops/system/config_envfile.sh
index 8acfb70959..dfa9490692 100755
--- a/bcs-ops/system/config_envfile.sh
+++ b/bcs-ops/system/config_envfile.sh
@@ -126,6 +126,7 @@ init_env() {
   ENABLE_APISERVER_HA=${ENABLE_APISERVER_HA:-"false"}
   APISERVER_HA_MODE=${APISERVER_HA_MODE:-"bcs-apiserver-proxy"}
   VIP=${VIP:-}
+  APISERVER_HOST=${APISERVER_HOST:-}
   ## bcs apiserver proxy
   APISERVER_PROXY_VERSION=${APISERVER_PROXY_VERSION:-"v1.29.0-alpha.130-tencent"}
   PROXY_TOOL_PATH=${PROXY_TOOL_PATH:-"/usr/bin"}
@@ -226,6 +227,12 @@ now is ${K8S_IPv6_STATUS}"
 
   [[ -n $K8S_CTRL_IP ]] || K8S_CTRL_IP=$LAN_IP
 
+  if [[ ${ENABLE_APISERVER_HA} == "external" ]];then
+    if [[ -z "${VIP}" ]] && [[ -z "${APISERVER_HOST}" ]];then
+      utils::log "ERROR" \
+        "if ENABLE_APISERVER_HA is ${ENABLE_APISERVER_HA}, VIP or APISERVER_HOST must be set"
+    fi
+  fi
 }
 
 render_env() {
@@ -323,6 +330,7 @@ BKREPO_URL="${BKREPO_URL}"
 ENABLE_APISERVER_HA="${ENABLE_APISERVER_HA}"
 APISERVER_HA_MODE="${APISERVER_HA_MODE}"
 VIP="${VIP}"
+APISERVER_HOST="${APISERVER_HOST}"
 ## bcs apiserver proxy
 APISERVER_PROXY_VERSION="${APISERVER_PROXY_VERSION}"
 PROXY_TOOL_PATH="${PROXY_TOOL_PATH}"