MGMT-16781: Implement ocp postprocess IP rename

run_seed.sh

@@ -102,6 +102,7 @@ use_cert_rules:
     -----END CERTIFICATE-----
 cluster_rename: new-name:foo.com:some-random-infra-id
 hostname: test.hostname
+ip: 192.168.126.99
 kubeadmin_password_hash: "$2a$10$20Q4iRLy7cWZkjn/D07bF.RZQZonKwstyRGH0qiYbYRkx5Pe4Ztyi"
 summary_file: summary.yaml
 summary_file_clean: summary_redacted.yaml
@@ -123,6 +124,7 @@ else
 		--cn-san-replace 192.168.126.10:192.168.127.11 \
 		--cluster-rename new-name:foo.com:some-random-infra-id \
 		--hostname test.hostname \
+		--ip 192.168.126.99 \
 		--kubeadmin-password-hash '$2a$10$20Q4iRLy7cWZkjn/D07bF.RZQZonKwstyRGH0qiYbYRkx5Pe4Ztyi' \
 		--summary-file summary.yaml \
 		--summary-file-clean summary_redacted.yaml \

src/config.rs

@@ -66,6 +66,7 @@ pub(crate) struct RecertConfig {
     pub(crate) customizations: Customizations,
     pub(crate) cluster_rename: Option<ClusterRenameParameters>,
     pub(crate) hostname: Option<String>,
+    pub(crate) ip: Option<String>,
     pub(crate) kubeadmin_password_hash: Option<String>,
     pub(crate) threads: Option<usize>,
     pub(crate) regenerate_server_ssh_keys: Option<ConfigPath>,
@@ -238,6 +239,11 @@ impl RecertConfig {
             None => None,
         };
 
+        let ip = match value.get("ip") {
+            Some(value) => Some(value.as_str().context("ip must be a string")?.to_string()),
+            None => None,
+        };
+
         let set_kubeadmin_password_hash = match value.get("kubeadmin_password_hash") {
             Some(value) => Some(value.as_str().context("set_kubeadmin_password_hash must be a string")?.to_string()),
             None => None,
@@ -298,6 +304,7 @@ impl RecertConfig {
             },
             cluster_rename,
             hostname,
+            ip,
             kubeadmin_password_hash: set_kubeadmin_password_hash,
             threads,
             regenerate_server_ssh_keys,
@@ -341,6 +348,7 @@ impl RecertConfig {
             },
             cluster_rename: cli.cluster_rename,
             hostname: cli.hostname,
+            ip: cli.ip,
             kubeadmin_password_hash: cli.kubeadmin_password_hash,
             threads: cli.threads,
             regenerate_server_ssh_keys: cli.regenerate_server_ssh_keys.map(ConfigPath::from),

src/config/cli.rs

@@ -40,6 +40,11 @@ pub(crate) struct Cli {
     #[clap(long)]
     pub(crate) hostname: Option<String>,
 
+    /// If given, the cluster resources that include the IP address will be modified to use this
+    /// one instead.
+    #[clap(long)]
+    pub(crate) ip: Option<String>,
+
     /// Modify the OCP kubeadmin password secret hash. If given but empty, the kubeadmin password
     /// secret will be deleted (thus disabling password login). If given and non-empty, the secret
     /// will be updated with the given password hash, unless no existing kubeadmin secret resource

src/ocp_postprocess.rs

@@ -18,12 +18,14 @@ use std::{collections::HashSet, sync::Arc};
 pub(crate) mod cluster_domain_rename;
 mod fnv;
 pub(crate) mod hostname_rename;
+pub(crate) mod ip_rename;
 
 /// Perform some OCP-related post-processing to make some OCP operators happy
 pub(crate) async fn ocp_postprocess(
     in_memory_etcd_client: &Arc<InMemoryK8sEtcd>,
     cluster_rename_params: &Option<ClusterRenameParameters>,
     hostname: &Option<String>,
+    ip: &Option<String>,
     kubeadmin_password_hash: &Option<String>,
     static_dirs: &Vec<ConfigPath>,
     static_files: &Vec<ConfigPath>,
@@ -56,6 +58,12 @@ pub(crate) async fn ocp_postprocess(
             .context("renaming hostname")?;
     }
 
+    if let Some(ip) = ip {
+        ip_rename(in_memory_etcd_client, ip, static_dirs, static_files)
+            .await
+            .context("renaming IP")?;
+    }
+
     if let Some(kubeadmin_password_hash) = kubeadmin_password_hash {
         log::info!("setting kubeadmin password hash");
         set_kubeadmin_password_hash(in_memory_etcd_client, kubeadmin_password_hash)
@@ -116,7 +124,7 @@ async fn set_kubeadmin_password_hash(in_memory_etcd_client: &InMemoryK8sEtcd, ku
                 ),
             );
 
-            put_etcd_yaml(etcd_client, k8s_resource_location, dbg!(secret)).await?;
+            put_etcd_yaml(etcd_client, k8s_resource_location, secret).await?;
 
             Ok(())
         }
@@ -431,3 +439,18 @@ pub(crate) async fn hostname_rename(
 
     Ok(())
 }
+
+pub(crate) async fn ip_rename(
+    in_memory_etcd_client: &Arc<InMemoryK8sEtcd>,
+    ip: &str,
+    static_dirs: &[ConfigPath],
+    static_files: &[ConfigPath],
+) -> Result<()> {
+    let etcd_client = in_memory_etcd_client;
+
+    ip_rename::rename_all(etcd_client, ip, static_dirs, static_files)
+        .await
+        .context("renaming all")?;
+
+    Ok(())
+}

src/ocp_postprocess/cluster_domain_rename/etcd_rename.rs

@@ -1,5 +1,5 @@
 use super::rename_utils::{
-    self, fix_api_server_arguments, fix_kcm_extended_args, fix_kcm_pod, fix_machineconfig, fix_oauth_metadata, fix_pod_container_env,
+    self, fix_api_server_arguments_domain, fix_kcm_extended_args, fix_kcm_pod, fix_machineconfig, fix_oauth_metadata, fix_pod_container_env,
 };
 use crate::{
     cluster_crypto::locations::K8sResourceLocation,
@@ -385,7 +385,7 @@ pub(crate) async fn fix_kube_apiserver_kubeapiserver(etcd_client: &Arc<InMemoryK
         .pointer_mut("/spec/observedConfig")
         .context("no /spec/observedConfig")?;
 
-    fix_api_server_arguments(config, cluster_domain).context("fixing config")?;
+    fix_api_server_arguments_domain(config, cluster_domain).context("fixing config")?;
 
     put_etcd_yaml(etcd_client, &k8s_resource_location, kubeapiserver).await?;
 
@@ -442,8 +442,6 @@ pub(crate) fn fix_openshift_apiserver_config(config: &mut Value, cluster_domain:
         )
         .context("missing subdomain")?;
 
-    // TODO: If we ever stop using a fake internal IP, we need to change .storageConfig.urls[0] to be the new IP here
-
     Ok(())
 }
 
@@ -904,7 +902,7 @@ pub(crate) async fn fix_kube_apiserver_configs(etcd_client: &Arc<InMemoryK8sEtcd
                     serde_yaml::from_slice(data["config.yaml"].as_str().context("config.yaml not a string")?.as_bytes())
                         .context("deserializing config.yaml")?;
 
-                fix_api_server_arguments(&mut config, cluster_domain)?;
+                fix_api_server_arguments_domain(&mut config, cluster_domain)?;
 
                 data.insert(
                     "config.yaml".to_string(),

src/ocp_postprocess/cluster_domain_rename/filesystem_rename.rs

@@ -1,5 +1,5 @@
 use super::rename_utils::{
-    fix_api_server_arguments, fix_apiserver_url_file, fix_kcm_extended_args, fix_kcm_pod, fix_kubeconfig, fix_machineconfig,
+    fix_api_server_arguments_domain, fix_apiserver_url_file, fix_kcm_extended_args, fix_kcm_pod, fix_kubeconfig, fix_machineconfig,
     fix_oauth_metadata,
 };
 use crate::file_utils::{self, commit_file, read_file_to_string};
@@ -101,7 +101,7 @@ pub(crate) async fn fix_filesystem_kube_apiserver_configs(cluster_domain: &str,
                             .context("reading kube-apiserver config.yaml")?;
                         let mut config: Value = serde_yaml::from_str(&contents).context("parsing kube-apiserver config.yaml")?;
 
-                        fix_api_server_arguments(&mut config, &cluster_domain)?;
+                        fix_api_server_arguments_domain(&mut config, &cluster_domain)?;
 
                         commit_file(
                             file_path,

src/ocp_postprocess/cluster_domain_rename/rename_utils.rs

@@ -60,7 +60,7 @@ pub(crate) fn fix_oauth_metadata(oauth_metadata: &mut Value, cluster_domain: &st
     Ok(())
 }
 
-pub(crate) fn fix_api_server_arguments(config: &mut Value, cluster_domain: &str) -> Result<()> {
+pub(crate) fn fix_api_server_arguments_domain(config: &mut Value, cluster_domain: &str) -> Result<()> {
     let apiserver_arguments = &mut config
         .pointer_mut("/apiServerArguments")
         .context("apiServerArguments not found")?
@@ -76,6 +76,36 @@ pub(crate) fn fix_api_server_arguments(config: &mut Value, cluster_domain: &str)
     Ok(())
 }
 
+pub(crate) fn fix_api_server_arguments_ip(config: &mut Value, original_ip: &str, ip: &str) -> Result<()> {
+    let apiserver_arguments = &mut config
+        .pointer_mut("/apiServerArguments")
+        .context("apiServerArguments not found")?
+        .as_object_mut()
+        .context("apiServerArguments not an object")?;
+
+    let original_etcd_servers = apiserver_arguments
+        .get("etcd-servers")
+        .context("etcd-servers not found")?
+        .as_array()
+        .context("etcd-servers not an array")?
+        .clone();
+
+    let new_etcd_servers = original_etcd_servers
+        .iter()
+        .map(|etcd_server| {
+            Ok(Value::String(etcd_server.as_str().context("etcd server not a string")?.replace(
+                format!("https://{original_ip}").as_str(),
+                format!("https://{ip}").as_str(),
+            )))
+        })
+        .collect::<Result<Vec<_>>>()
+        .context("replacing etcd servers")?;
+
+    apiserver_arguments.insert("etcd-servers".to_string(), Value::Array(new_etcd_servers));
+
+    Ok(())
+}
+
 /// Based on https://github.com/openshift/installer/blob/e91df626c10e569e7613249053b7b9b264db42df/pkg/asset/installconfig/clusterid.go#L57-L79
 pub(crate) fn generate_infra_id(cluster_name: &str) -> Result<String> {
     const CLUSTER_INFRA_ID_RANDOM_LEN: usize = 5;
@@ -327,7 +357,7 @@ pub(crate) fn env_var_safe(node_name: &str) -> String {
     node_name.replace(['-', '.'], "_")
 }
 
-pub(crate) fn fix_etcd_pod_yaml(pod_yaml: &str, original_hostname: &str, hostname: &str) -> Result<String> {
+pub(crate) fn fix_etcd_pod_yaml_hostname(pod_yaml: &str, original_hostname: &str, hostname: &str) -> Result<String> {
     let mut pod_yaml = pod_yaml.to_string();
 
     // TODO: The "value:" replacement below is risky - if the hostname is "existing",
@@ -406,6 +436,23 @@ pub(crate) fn fix_etcd_pod_yaml(pod_yaml: &str, original_hostname: &str, hostnam
     Ok(pod_yaml)
 }
 
+pub(crate) fn fix_etcd_pod_yaml_ip(pod_yaml: &str, original_ip: &str, ip: &str) -> Result<String> {
+    let mut pod_yaml = pod_yaml.to_string();
+
+    let patterns = [
+        (r#"value: "https://{original_ip}:2379""#, r#"value: "https://{ip}:2379""#),
+        (r#"value: "{original_ip}""#, r#"value: "{ip}""#),
+    ];
+
+    for (pattern, replacement) in patterns {
+        pod_yaml = pod_yaml
+            .replace(&pattern.replace("{original_ip}", original_ip), &replacement.replace("{ip}", ip))
+            .to_string();
+    }
+
+    Ok(pod_yaml)
+}
+
 pub(crate) fn fix_etcd_static_pod(pod: &mut Value, original_hostname: &str, hostname: &str) -> Result<()> {
     {
         let init_containers = &mut pod

src/ocp_postprocess/hostname_rename/etcd_rename.rs

@@ -1,7 +1,7 @@
 use crate::{
     cluster_crypto::locations::K8sResourceLocation,
     k8s_etcd::{get_etcd_json, put_etcd_yaml, InMemoryK8sEtcd},
-    ocp_postprocess::cluster_domain_rename::rename_utils::{env_var_safe, fix_etcd_pod_yaml},
+    ocp_postprocess::cluster_domain_rename::rename_utils::{env_var_safe, fix_etcd_pod_yaml_hostname},
 };
 use anyhow::{ensure, Context, Result};
 use futures_util::future::join_all;
@@ -271,7 +271,7 @@ pub(crate) async fn fix_etcd_pod(etcd_client: &Arc<InMemoryK8sEtcd>, original_ho
                     .context("pod.yaml not a string")?
                     .to_string();
 
-                let pod_yaml = fix_etcd_pod_yaml(&pod_yaml, original_hostname, hostname).context("could not fix pod yaml")?;
+                let pod_yaml = fix_etcd_pod_yaml_hostname(&pod_yaml, original_hostname, hostname).context("could not fix pod yaml")?;
 
                 data.insert("pod.yaml".to_string(), serde_json::Value::String(pod_yaml));
 

src/ocp_postprocess/hostname_rename/filesystem_rename.rs

@@ -50,7 +50,7 @@ pub(crate) async fn fix_filesystem_etcd_configmap_pod_yaml(original_hostname: &s
 
                 commit_file(
                     file_path,
-                    rename_utils::fix_etcd_pod_yaml(&contents, &original_hostname, &hostname).context("fixing etcd-pod.yaml")?,
+                    rename_utils::fix_etcd_pod_yaml_hostname(&contents, &original_hostname, &hostname).context("fixing etcd-pod.yaml")?,
                 )
                 .await
                 .context("writing etcd-pod.yaml to disk")?;

src/ocp_postprocess/ip_rename.rs

@@ -0,0 +1,94 @@
+use crate::{config::ConfigPath, k8s_etcd::InMemoryK8sEtcd};
+use anyhow::{Context, Result};
+use std::{path::Path, sync::Arc};
+
+mod etcd_rename;
+mod filesystem_rename;
+
+pub(crate) async fn rename_all(
+    etcd_client: &Arc<InMemoryK8sEtcd>,
+    ip: &str,
+    static_dirs: &[ConfigPath],
+    static_files: &[ConfigPath],
+) -> Result<(), anyhow::Error> {
+    let original_ip = fix_etcd_resources(etcd_client, ip).await.context("renaming etcd resources")?;
+
+    fix_filesystem_resources(&original_ip, ip, static_dirs, static_files)
+        .await
+        .context("renaming filesystem resources")?;
+
+    Ok(())
+}
+
+async fn fix_filesystem_resources(
+    original_ip: &str,
+    ip: &str,
+    static_dirs: &[ConfigPath],
+    static_files: &[ConfigPath],
+) -> Result<(), anyhow::Error> {
+    for dir in static_dirs {
+        fix_dir_resources(original_ip, ip, dir).await?;
+    }
+
+    for file in static_files {
+        fix_file_resources(original_ip, ip, file).await?;
+    }
+
+    Ok(())
+}
+
+async fn fix_dir_resources(_original_ip: &str, _ip: &str, _dir: &Path) -> Result<()> {
+    // TODO: This is currently achieved using:
+    // https://github.com/openshift-kni/lifecycle-agent/blob/3f447f629cf73a25a350c1c2cc88d95bf2a31956/lca-cli/postpivot/postpivot.go#L232-L236
+    //
+    // But it should be done using recert instead at some point.
+    Ok(())
+}
+
+async fn fix_file_resources(_original_ip: &str, _ip: &str, _file: &Path) -> Result<()> {
+    Ok(())
+}
+
+async fn fix_etcd_resources(etcd_client: &Arc<InMemoryK8sEtcd>, ip: &str) -> Result<String> {
+    let original_ip = etcd_rename::fix_openshift_apiserver_configmap(etcd_client, ip)
+        .await
+        .context("fixing openshift apiserver config configmap")?;
+
+    etcd_rename::fix_etcd_endpoints(etcd_client, ip)
+        .await
+        .context("fixing etcd secrets")?;
+
+    etcd_rename::fix_etcd_pod(etcd_client, &original_ip, ip)
+        .await
+        .context("fixing etcd-pod")?;
+
+    etcd_rename::fix_etcd_scripts(etcd_client, &original_ip, ip)
+        .await
+        .context("fixing etcd-scripts")?;
+
+    etcd_rename::fix_kube_apiserver_configs(etcd_client, &original_ip, ip)
+        .await
+        .context("fixing kube apiserver configs")?;
+
+    etcd_rename::fix_kubeapiservers_cluster(etcd_client, &original_ip, ip)
+        .await
+        .context("fixing kubeapiservers/cluster")?;
+
+    etcd_rename::fix_authentications_cluster(etcd_client, &original_ip, ip)
+        .await
+        .context("fixing kubeapiservers/cluster")?;
+
+    etcd_rename::fix_openshiftapiservers_cluster(etcd_client, ip)
+        .await
+        .context("fixing kubeapiservers/cluster")?;
+
+    etcd_rename::fix_networks_cluster(etcd_client, ip)
+        .await
+        .context("fixing networks/cluster")?;
+
+    etcd_rename::fix_oauth_apiserver_deployment(etcd_client, &original_ip, ip)
+        .await
+        .context("fixing oauth apiserver deployment")?;
+
+    Ok(original_ip)
+}