diff --git a/run_seed.sh b/run_seed.sh index e6abce6e..556a678d 100755 --- a/run_seed.sh +++ b/run_seed.sh @@ -93,6 +93,7 @@ use_cert_rules: BgIejfD1dYW2Fp02z5sF6Pw6vhobpfDYgsTAKNonh5P6NxMiD14eQxYrNJ6DAF0= -----END CERTIFICATE----- cluster_rename: new-name:foo.com:some-random-infra-id +hostname: test.hostname summary_file: summary.yaml summary_file_clean: summary_redacted.yaml extend_expiration: true @@ -111,6 +112,7 @@ else --cn-san-replace *.apps.seed.redhat.com:*.apps.new-name.foo.com \ --cn-san-replace 192.168.126.10:192.168.127.11 \ --cluster-rename new-name:foo.com:some-random-infra-id \ + --hostname test.hostname \ --summary-file summary.yaml \ --summary-file-clean summary_redacted.yaml \ --extend-expiration diff --git a/src/config.rs b/src/config.rs index 57e1e1b8..6e9bc6c8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -65,6 +65,7 @@ pub(crate) struct RecertConfig { pub(crate) static_files: Vec, pub(crate) customizations: Customizations, pub(crate) cluster_rename: Option, + pub(crate) hostname: Option, pub(crate) threads: Option, pub(crate) regenerate_server_ssh_keys: Option, pub(crate) summary_file: Option, @@ -231,6 +232,11 @@ impl RecertConfig { None => None, }; + let hostname = match value.get("hostname") { + Some(value) => Some(value.as_str().context("hostname must be a string")?.to_string()), + None => None, + }; + let threads = match value.get("threads") { Some(value) => Some( value @@ -285,6 +291,7 @@ impl RecertConfig { force_expire, }, cluster_rename, + hostname, threads, regenerate_server_ssh_keys, summary_file, @@ -326,6 +333,7 @@ impl RecertConfig { force_expire: cli.force_expire, }, cluster_rename: cli.cluster_rename, + hostname: cli.hostname, threads: cli.threads, regenerate_server_ssh_keys: cli.regenerate_server_ssh_keys.map(ConfigPath::from), summary_file: cli.summary_file.map(ConfigPath::from), diff --git a/src/config/cli.rs b/src/config/cli.rs index 4903062a..292f176e 100644 --- a/src/config/cli.rs +++ b/src/config/cli.rs @@ -35,6 +35,11 @@ pub(crate) struct Cli { #[clap(long, value_parser = ClusterRenameParameters::cli_parse)] pub(crate) cluster_rename: Option, + /// If given, the cluster resources that include the hostname will be modified to use this one + /// instead. + #[clap(long)] + pub(crate) hostname: Option, + /// A list of CNs and the private keys to use for their certs. By default, new keys will be /// generated for all regenerated certificates, this option allows you to use existing keys /// instead. Must come in pairs of CN and private key file path, separated by a space. For diff --git a/src/ocp_postprocess.rs b/src/ocp_postprocess.rs index 69d1096b..34e7fbde 100644 --- a/src/ocp_postprocess.rs +++ b/src/ocp_postprocess.rs @@ -17,11 +17,13 @@ use std::{collections::HashSet, sync::Arc}; pub(crate) mod cluster_domain_rename; mod fnv; +pub(crate) mod hostname_rename; /// Perform some OCP-related post-processing to make some OCP operators happy pub(crate) async fn ocp_postprocess( in_memory_etcd_client: &Arc, cluster_rename_params: &Option, + hostname: &Option, static_dirs: &Vec, static_files: &Vec, ) -> Result<()> { @@ -47,6 +49,12 @@ pub(crate) async fn ocp_postprocess( .context("renaming cluster")?; } + if let Some(hostname) = hostname { + hostname_rename(in_memory_etcd_client, hostname, static_dirs, static_files) + .await + .context("renaming hostname")?; + } + fix_deployment_dep_annotations( in_memory_etcd_client, K8sResourceLocation::new(Some("openshift-apiserver"), "Deployment", "apiserver", "v1"), @@ -357,3 +365,18 @@ pub(crate) async fn cluster_rename( Ok(()) } + +pub(crate) async fn hostname_rename( + in_memory_etcd_client: &Arc, + hostname: &str, + static_dirs: &[ConfigPath], + static_files: &[ConfigPath], +) -> Result<()> { + let etcd_client = in_memory_etcd_client; + + hostname_rename::rename_all(etcd_client, hostname, static_dirs, static_files) + .await + .context("renaming all")?; + + Ok(()) +} diff --git a/src/ocp_postprocess/cluster_domain_rename.rs b/src/ocp_postprocess/cluster_domain_rename.rs index 67a65e32..6d58814e 100644 --- a/src/ocp_postprocess/cluster_domain_rename.rs +++ b/src/ocp_postprocess/cluster_domain_rename.rs @@ -6,7 +6,7 @@ use std::{path::Path, sync::Arc}; mod etcd_rename; mod filesystem_rename; pub(crate) mod params; -mod rename_utils; +pub(crate) mod rename_utils; pub(crate) async fn rename_all( etcd_client: &Arc, diff --git a/src/ocp_postprocess/cluster_domain_rename/filesystem_rename.rs b/src/ocp_postprocess/cluster_domain_rename/filesystem_rename.rs index 547997d8..7a3dca46 100644 --- a/src/ocp_postprocess/cluster_domain_rename/filesystem_rename.rs +++ b/src/ocp_postprocess/cluster_domain_rename/filesystem_rename.rs @@ -1,10 +1,6 @@ -use super::{ - rename_utils::fix_api_server_arguments, - rename_utils::fix_apiserver_url_file, - rename_utils::fix_kcm_extended_args, - rename_utils::fix_kubeconfig, - rename_utils::fix_oauth_metadata, - rename_utils::{fix_kcm_pod, fix_machineconfig}, +use super::rename_utils::{ + fix_api_server_arguments, fix_apiserver_url_file, fix_kcm_extended_args, fix_kcm_pod, fix_kubeconfig, fix_machineconfig, + fix_oauth_metadata, }; use crate::file_utils::{self, commit_file, read_file_to_string}; use anyhow::{self, Context, Result}; diff --git a/src/ocp_postprocess/cluster_domain_rename/rename_utils.rs b/src/ocp_postprocess/cluster_domain_rename/rename_utils.rs index 1fe8fc99..2318058e 100644 --- a/src/ocp_postprocess/cluster_domain_rename/rename_utils.rs +++ b/src/ocp_postprocess/cluster_domain_rename/rename_utils.rs @@ -122,6 +122,49 @@ pub(crate) fn fix_kcm_extended_args(config: &mut Value, generated_infra_id: &str Ok(()) } +pub(crate) fn fix_cluster_backup_sh(cluster_backup_sh: &str, original_hostname: &str, hostname: &str) -> Result { + let cluster_backup = cluster_backup_sh.to_string(); + let pattern = format!(r"NODE_{original_hostname}_IP"); + let replacement = format!(r"NODE_{}_IP", env_var_safe(hostname)); + Ok(cluster_backup.replace(&pattern, &replacement)) +} + +pub(crate) fn fix_etcd_env(etcd_env: &str, original_hostname: &str, hostname: &str) -> Result { + let mut etcd_env = etcd_env.to_string(); + let patterns = [ + (r#"NODE_{original_hostname_safe}_IP"#, r#"NODE_{hostname_safe}_IP"#), + ( + r#"NODE_{original_hostname_safe}_ETCD_NAME="{original_hostname}""#, + r#"NODE_{hostname_safe}_ETCD_NAME="{hostname}""#, + ), + ( + r#"NODE_{original_hostname_safe}_ETCD_URL_HOST"#, + r#"NODE_{hostname_safe}_ETCD_URL_HOST"#, + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{original_hostname}.crt", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{hostname}.crt", + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{original_hostname}.key", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{hostname}.key", + ), + ]; + for (pattern, replacement) in patterns { + let pattern = pattern + .replace("{original_hostname}", original_hostname) + .replace("{original_hostname_safe}", &env_var_safe(original_hostname)); + + let replacement = replacement + .replace("{hostname}", hostname) + .replace("{hostname_safe}", &env_var_safe(hostname)); + + etcd_env = etcd_env.replace(&pattern, &replacement).to_string(); + } + + Ok(etcd_env) +} + pub(crate) async fn fix_kubeconfig(cluster_name: &str, cluster_domain: &str, kubeconfig: &mut Value) -> Result<()> { let is_kubelet_kubeconfig = kubeconfig .pointer_mut("/contexts") @@ -279,6 +322,267 @@ pub(crate) fn fix_kcm_pod(pod: &mut Value, generated_infra_id: &str) -> Result<( Ok(()) } +// Mimics https://github.com/openshift/cluster-etcd-operator/blob/5973046e2d216b290740cf64a071a272bbf83aea/pkg/etcdenvvar/etcd_env.go#L244-L246 +pub(crate) fn env_var_safe(node_name: &str) -> String { + node_name.replace(['-', '.'], "_") +} + +pub(crate) fn fix_etcd_pod_yaml(pod_yaml: &str, original_hostname: &str, hostname: &str) -> Result { + let mut pod_yaml = pod_yaml.to_string(); + + // TODO: The "value:" replacement below is risky - if the hostname is "existing", + // or "REVISION", or "true" this will wreak havoc because these appear in the + // pod.yaml as values. Unlikely but crash if we see these values for now. + ensure!( + ["existing", "REVISION", "true"] + .iter() + .all(|invalid_hostname| invalid_hostname != &original_hostname), + "{} hostname is unsupported at the moment, please use a different seed hostname", + original_hostname + ); + + let patterns = [ + ( + r#"- name: "NODE_{original_hostname_safe}_ETCD_NAME"#, + r#"- name: "NODE_{hostname_safe}_ETCD_NAME"#, + ), + (r#"value: "{original_hostname}""#, r#"value: "{hostname}""#), + ( + r#"- name: "NODE_{original_hostname_safe}_ETCD_URL_HOST"#, + r#"- name: "NODE_{hostname_safe}_ETCD_URL_HOST"#, + ), + ( + r#"- name: "NODE_{original_hostname_safe}_IP"#, + r#"- name: "NODE_{hostname_safe}_IP"#, + ), + ( + r#"${NODE_{original_hostname_safe}_ETCD_URL_HOST"#, + r#"${NODE_{hostname_safe}_ETCD_URL_HOST"#, + ), + ( + r#"${NODE_{original_hostname_safe}_ETCD_NAME"#, + r#"${NODE_{hostname_safe}_ETCD_NAME""#, + ), + ("${NODE_{original_hostname_safe}_IP", "${NODE_{hostname_safe}_IP"), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{original_hostname}.crt", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{hostname}.crt", + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{original_hostname}.key", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{hostname}.key", + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-metrics-{original_hostname}.crt", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-metrics-{hostname}.crt", + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-metrics-{original_hostname}.key", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-metrics-{hostname}.key", + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-{original_hostname}.key", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-{hostname}.key", + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-{original_hostname}.crt", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-{hostname}.crt", + ), + ("--target-name={original_hostname}", "--target-name={hostname}"), + ]; + + for (pattern, replacement) in patterns { + let pattern = pattern + .replace("{original_hostname}", original_hostname) + .replace("{original_hostname_safe}", &env_var_safe(original_hostname)); + + let replacement = replacement + .replace("{hostname}", hostname) + .replace("{hostname_safe}", &env_var_safe(hostname)); + + pod_yaml = pod_yaml.replace(&pattern, &replacement).to_string(); + } + + Ok(pod_yaml) +} + +pub(crate) fn fix_etcd_static_pod(pod: &mut Value, original_hostname: &str, hostname: &str) -> Result<()> { + { + let init_containers = &mut pod + .pointer_mut("/spec/initContainers") + .context("initContainers not found")? + .as_array_mut() + .context("initContainers not an object")?; + + ensure!(!init_containers.is_empty(), "expected at least one init container in pod.yaml"); + + init_containers + .iter_mut() + .try_for_each(|container| fix_etcd_static_pod_container(container, original_hostname, hostname))?; + } + + { + let containers = &mut pod + .pointer_mut("/spec/containers") + .context("containers not found")? + .as_array_mut() + .context("containers not an object")?; + + ensure!(!containers.is_empty(), "expected at least one container in pod.yaml"); + + containers + .iter_mut() + .try_for_each(|container| { + fix_etcd_static_pod_container(container, original_hostname, hostname) + .context(format!("fixing container {}", container.get("name").unwrap_or(&Value::Null))) + }) + .context("fixing etcd static pod container")?; + } + + Ok(()) +} + +fn fix_etcd_static_pod_container(container: &mut Value, original_hostname: &str, hostname: &str) -> Result<()> { + 'hostname_args_replace: { + let args = container + .pointer_mut("/command") + .context("command not found")? + .as_array_mut() + .context("command not an array")?; + + ensure!(!args.is_empty(), "expected at least one arg in etcd static pod container"); + + let shell_arg = args + .iter_mut() + .find_map(|arg| arg.as_str()?.starts_with("#!/bin/sh\n").then_some(arg)); + + let shell_arg = match shell_arg { + None => break 'hostname_args_replace, + Some(shell_arg) => shell_arg, + }; + + for (original, new) in [ + ("NODE_{original_hostname}_ETCD_URL_HOST", "NODE_{hostname}_ETCD_URL_HOST"), + ("NODE_{original_hostname}_ETCD_NAME", "NODE_{hostname}_ETCD_NAME"), + ("NODE_{original_hostname}_IP", "NODE_{hostname}_ETCD_LISTEN_CLIENT_URLS"), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{original_hostname}.crt", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{hostname}.crt", + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{original_hostname}.key", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{hostname}.key", + ), + ("--target-name={original_hostname}", "--target-name={hostname}"), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-{original_hostname}.crt", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-{hostname}.crt", + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-{original_hostname}.key", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-{hostname}.key", + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-metrics-{original_hostname}.crt", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-metrics-{hostname}.crt", + ), + ( + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-metrics-{original_hostname}.key", + "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-metrics-{hostname}.key", + ), + ] { + *shell_arg = serde_json::Value::String( + regex::Regex::new(original.replace("{original_hostname}", original_hostname).as_str()) + .unwrap() + .replace_all( + shell_arg.as_str().context("arg not string")?, + new.replace("{hostname}", hostname).as_str(), + ) + .to_string(), + ); + } + } + + 'hostname_env_replace: { + let maybe_env = container.pointer_mut("/env"); + + let envs = match maybe_env { + Some(env) => env.as_array_mut().context("env not an array")?, + None => break 'hostname_env_replace, + }; + + ensure!(!envs.is_empty(), "expected at least one env in etcd static pod container"); + + for (key, new_name, new_value) in [ + ( + "ETCDCTL_CERT", + None, + Some(format!("/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{hostname}.crt").as_str()), + ), + ( + "ETCDCTL_KEY", + None, + Some(format!("/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{hostname}.key").as_str()), + ), + ( + "ETCDCTL_KEY_FILE", + None, + Some(format!("/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-{hostname}.key").as_str()), + ), + ( + format!("NODE_{original_hostname}_ETCD_NAME").as_str(), + Some(format!("NODE_{hostname}_ETCD_NAME").as_str()), + Some(hostname), + ), + ( + format!("NODE_{original_hostname}_ETCD_URL_HOST").as_str(), + Some(format!("NODE_{hostname}_ETCD_URL_HOST").as_str()), + None, + ), + ( + format!("NODE_{original_hostname}_IP").as_str(), + Some(format!("NODE_{hostname}_IP").as_str()), + None, + ), + ] { + adjust_env(envs, key, new_name, new_value).context(format!("adjusting env var {}", key))?; + } + } + + Ok(()) +} + +fn adjust_env(envs: &mut [Value], env_name: &str, new_name: Option<&str>, new_value: Option<&str>) -> Result<()> { + let found_env = envs + .iter_mut() + .find_map(|env| (env.as_object()?.get("name") == Some(&Value::String(env_name.to_string()))).then_some(env)); + + match found_env { + None => Ok(()), + Some(env) => { + match new_name { + None => {} + Some(new_name) => { + env.as_object_mut() + .context("env var not an object")? + .insert("name".to_string(), serde_json::Value::String(new_name.to_string())); + } + }; + + match new_value { + None => {} + Some(new_value) => { + env.as_object_mut() + .context("env var not an object")? + .insert("value".to_string(), serde_json::Value::String(new_value.to_string())); + } + }; + + Ok(()) + } + } +} + pub(crate) fn fix_pod_container_env(pod: &mut Value, domain: &str, container_name: &str, env_name: &str, init: bool) -> Result<()> { let containers = &mut pod .pointer_mut(&format!("/spec/{}", if init { "initContainers" } else { "containers" })) @@ -404,3 +708,47 @@ pub(crate) fn fix_machineconfig(machineconfig: &mut Value, cluster_domain: &str) Ok(()) } + +pub(crate) fn fix_kapi_startup_monitor_pod_container_args(pod: &mut Value, hostname: &str) -> Result<()> { + let containers = &mut pod + .pointer_mut("/spec/containers") + .context("containers not found")? + .as_array_mut() + .context("containers not an object")?; + + if containers.is_empty() { + bail!("expected at least one container in pod.yaml"); + } + + containers + .iter_mut() + .filter(|container| container["name"] == "startup-monitor") + .try_for_each(|container| { + let args = container + .pointer_mut("/args") + .context("args not found")? + .as_array_mut() + .context("args not an array")?; + + ensure!(!args.is_empty(), "expected at least one arg in container"); + + let arg_idx = args + .iter_mut() + .enumerate() + .find_map(|(i, arg)| arg.as_str()?.starts_with("--node-name=").then_some(i)) + .context("--node-name not found")?; + + args[arg_idx] = serde_json::Value::String(format!("--node-name={}", hostname)); + + Ok(()) + })?; + + Ok(()) +} + +pub(crate) fn fix_kapi_startup_monitor_pod_yaml(pod_yaml: &str, original_hostname: &str, hostname: &str) -> Result { + let pod_yaml = pod_yaml.to_string(); + let pattern = format!(r"--node-name={}", original_hostname); + let replacement = format!(r"--node-name={}", hostname); + Ok(pod_yaml.replace(&pattern, &replacement)) +} diff --git a/src/ocp_postprocess/hostname_rename.rs b/src/ocp_postprocess/hostname_rename.rs new file mode 100644 index 00000000..ad53c410 --- /dev/null +++ b/src/ocp_postprocess/hostname_rename.rs @@ -0,0 +1,96 @@ +use crate::{config::ConfigPath, k8s_etcd::InMemoryK8sEtcd}; +use anyhow::{Context, Result}; +use std::{path::Path, sync::Arc}; + +mod etcd_rename; +mod filesystem_rename; + +pub(crate) async fn rename_all( + etcd_client: &Arc, + hostname: &str, + static_dirs: &[ConfigPath], + static_files: &[ConfigPath], +) -> Result<(), anyhow::Error> { + let original_hostname = fix_etcd_resources(etcd_client, hostname).await.context("renaming etcd resources")?; + + fix_filesystem_resources(&original_hostname, hostname, static_dirs, static_files) + .await + .context("renaming filesystem resources")?; + + Ok(()) +} + +async fn fix_filesystem_resources( + original_hostname: &str, + hostname: &str, + static_dirs: &[ConfigPath], + static_files: &[ConfigPath], +) -> Result<(), anyhow::Error> { + for dir in static_dirs { + fix_dir_resources(original_hostname, hostname, dir).await?; + } + + for file in static_files { + fix_file_resources(original_hostname, hostname, file).await?; + } + + Ok(()) +} + +async fn fix_dir_resources(original_hostname: &str, hostname: &str, dir: &Path) -> Result<()> { + filesystem_rename::fix_filesystem_etcd_static_pods(original_hostname, hostname, dir) + .await + .context("fixing etcd static pods")?; + + filesystem_rename::fix_filesystem_etcd_configmap_pod_yaml(original_hostname, hostname, dir) + .await + .context("fixing etcd static pod configmap pod yaml")?; + + filesystem_rename::fix_filesystem_etcd_scripts_cluster_backup_sh(original_hostname, hostname, dir) + .await + .context("fixing etcd scripts cluster-backup.sh")?; + + filesystem_rename::fix_filesystem_etcd_scripts_etcd_env(original_hostname, hostname, dir) + .await + .context("fixing etcd scripts etcd.env")?; + + filesystem_rename::fix_filesystem_kapi_startup_monitor_pod(hostname, dir) + .await + .context("fixing kube-apiserver-startup-monitor-pod")?; + + filesystem_rename::fix_filesystem_kapi_startup_monitor_configmap_pod_yaml(original_hostname, hostname, dir) + .await + .context("fixing kube-apiserver-startup-monitor-pod configmap pod yaml")?; + + Ok(()) +} + +async fn fix_file_resources(_original_hostname: &str, _hostname: &str, _file: &Path) -> Result<()> { + Ok(()) +} + +async fn fix_etcd_resources(etcd_client: &Arc, hostname: &str) -> Result { + let original_hostname = etcd_rename::fix_etcd_all_certs(etcd_client, hostname) + .await + .context("fixing etcd-all-certs")?; + etcd_rename::fix_etcd_secrets(etcd_client, &original_hostname, hostname) + .await + .context("fixing etcd secrets")?; + etcd_rename::fix_etcd_pod(etcd_client, &original_hostname, hostname) + .await + .context("fixing etcd-pod")?; + etcd_rename::fix_etcd_scripts(etcd_client, &original_hostname, hostname) + .await + .context("fixing etcd-scripts")?; + etcd_rename::fix_kubeapiservers_cluster(etcd_client, hostname) + .await + .context("fixing kubeapiservers/cluster")?; + etcd_rename::fix_kubeschedulers_cluster(etcd_client, hostname) + .await + .context("fixing kubeschedulers/cluster")?; + etcd_rename::fix_kubecontrollermanagers_cluster(etcd_client, hostname) + .await + .context("fixing kubecontrollermanagers/cluster")?; + + Ok(original_hostname) +} diff --git a/src/ocp_postprocess/hostname_rename/etcd_rename.rs b/src/ocp_postprocess/hostname_rename/etcd_rename.rs new file mode 100644 index 00000000..4f6604d2 --- /dev/null +++ b/src/ocp_postprocess/hostname_rename/etcd_rename.rs @@ -0,0 +1,392 @@ +use crate::{ + cluster_crypto::locations::K8sResourceLocation, + k8s_etcd::{get_etcd_json, put_etcd_yaml, InMemoryK8sEtcd}, + ocp_postprocess::cluster_domain_rename::rename_utils::{env_var_safe, fix_etcd_pod_yaml}, +}; +use anyhow::{ensure, Context, Result}; +use futures_util::future::join_all; +use serde_json::{Map, Value}; +use std::{collections::HashSet, fmt::Display, sync::Arc}; + +async fn fix_etcd_all_certs_secret(etcd_client: &Arc, key: &str, hostname: &str) -> Result> { + let etcd_result = etcd_client + .get(key.to_string()) + .await + .with_context(|| format!("getting key {:?}", key))? + .context("key disappeared")?; + let value: Value = + serde_yaml::from_slice(etcd_result.value.as_slice()).with_context(|| format!("deserializing value of key {:?}", key,))?; + let k8s_resource_location = K8sResourceLocation::try_from(&value)?; + + let mut secret = get_etcd_json(etcd_client, &k8s_resource_location) + .await? + .context("getting secret")?; + + let original_hostname = { + let data = &mut secret + .pointer_mut("/data") + .context("no /data")? + .as_object_mut() + .context("data not an object")?; + + match data + .iter() + // etcd-peer is the only key that we can use to unambiguously identify the original hostname + .find(|(k, _v)| k.starts_with("etcd-peer-") && k.ends_with(".crt")) + .map(|(k, _v)| k.clone()) + { + Some(data_key) => data_key.trim_start_matches("etcd-peer-").trim_end_matches(".crt").to_string(), + None => { + // This is OK, some of the secrets don't have data keys + return anyhow::Ok(None); + } + } + }; + + let data_prefixes = ["etcd-peer-", "etcd-serving-", "etcd-serving-metrics-"]; + let managed_fields_data_prefixes = data_prefixes.iter().map(|prefix| format!("f:{}", prefix)).collect::>(); + + let suffixes = [".crt", ".key"]; + + fn replace_keys( + original_hostname: &str, + new_hostname: &str, + data_prefixes: &[impl Display], + suffixes: &[impl Display], + data: &mut &mut Map, + ) -> Result<()> { + let old_keys = data_prefixes.iter().flat_map(|prefix| { + suffixes + .iter() + .map(move |suffix| format!("{}{}{}", prefix, original_hostname, suffix)) + }); + + let new_keys = data_prefixes + .iter() + .flat_map(|prefix| suffixes.iter().map(move |suffix| format!("{}{}{}", prefix, new_hostname, suffix))); + + old_keys.zip(new_keys).for_each(|(old_key, new_key)| { + let value = data.remove(&old_key).context(format!("could not remove key: {}", old_key)).unwrap(); + data.insert(new_key, value); + }); + + Ok(()) + } + + // Adjust .data + { + let data = &mut secret + .pointer_mut("/data") + .context("no /data")? + .as_object_mut() + .context("data not an object")?; + + replace_keys(&original_hostname, hostname, &data_prefixes, &suffixes, data).context("could not replace keys")?; + } + + // Adjust .metadata.managedFields.fieldsV1.data + { + let metadata = &mut secret + .pointer_mut("/metadata") + .context("no /metadata")? + .as_object_mut() + .context("data not an object")?; + + let managed_fields = metadata + .get_mut("managedFields") + .context("no managedFields")? + .as_array_mut() + .context("managedFields not an array")?; + + managed_fields.iter_mut().try_for_each(|managed_field| { + let fields_v1_raw_byte_array = managed_field + .pointer("/fieldsV1/raw") + .context("no /fieldsV1/raw")? + .as_array() + .context("/fieldsV1/raw not an array")?; + + let mut fields_v1_raw_parsed: Value = serde_json::from_str( + &String::from_utf8( + fields_v1_raw_byte_array + .iter() + .map(|v| v.as_u64().context("fieldsV1 not a number")) + .collect::>>() + .context("parsing byte array")? + .into_iter() + .map(|v| v as u8) + .collect::>(), + ) + .context("fieldsV1 not valid utf8")?, + ) + .context("deserializing fieldsV1")?; + + let mut data = (match fields_v1_raw_parsed.pointer_mut("/f:data") { + Some(data) => data, + None => return anyhow::Ok(()), + }) + .as_object_mut() + .context("f:data not an object")?; + + replace_keys(&original_hostname, hostname, &managed_fields_data_prefixes, &suffixes, &mut data) + .context("could not replace managed fields keys")?; + + managed_field + .pointer_mut("/fieldsV1") + .context("no /fieldsV1")? + .as_object_mut() + .context("/fieldsV1 not an object")? + .insert( + "raw".to_string(), + serde_json::Value::Array( + serde_json::Value::String(serde_json::to_string(&fields_v1_raw_parsed).context("serializing fieldsV1")?) + .as_str() + .context("serialized not a string")? + .as_bytes() + .iter() + .map(|b| serde_json::Value::Number(serde_json::Number::from(*b))) + .collect(), + ), + ); + + anyhow::Ok(()) + })?; + } + + put_etcd_yaml(etcd_client, &k8s_resource_location, secret) + .await + .context(format!("could not put etcd key: {}", key))?; + + Ok(Some(original_hostname)) +} + +pub(crate) async fn fix_etcd_all_certs(etcd_client: &Arc, hostname: &str) -> Result { + let hostnames = join_all( + etcd_client + .list_keys("secrets/openshift-etcd/etcd-all-certs") + .await? + .into_iter() + .map(|key| async move { anyhow::Ok(fix_etcd_all_certs_secret(etcd_client, &key, hostname).await?) }), + ) + .await + .into_iter() + .collect::>>>()?; + + let hostnames = hostnames.into_iter().flatten().collect::>(); + + ensure!( + hostnames.len() == 1, + "no hostnames or multiple hostnames found in etcd-all-certs secrets: {:?}", + hostnames + ); + + let original_hostname = hostnames.into_iter().next().unwrap(); + + Ok(original_hostname) +} + +pub(crate) async fn fix_etcd_secrets(etcd_client: &Arc, original_hostname: &str, hostname: &str) -> Result<()> { + for key_prefix in ["etcd-peer", "etcd-serving", "etcd-serving-metrics"] { + join_all( + etcd_client + .list_keys(format!("secrets/openshift-etcd/{key_prefix}-{original_hostname}").as_str()) + .await? + .into_iter() + .map(|key| async move { + let etcd_result = etcd_client + .get(key.clone()) + .await + .with_context(|| format!("getting key {key:?}"))? + .context("key disappeared")?; + + let mut etcd_value: Value = serde_yaml::from_slice(etcd_result.value.as_slice()).context("deserializing value")?; + + let new_secret_name = format!("{key_prefix}-{hostname}"); + + etcd_value + .pointer_mut("/metadata") + .context("no /metadata")? + .as_object_mut() + .context("/metadata not an object")? + .insert("name".to_string(), serde_json::Value::String(new_secret_name.clone())); + + etcd_client + .put( + &(format!("/kubernetes.io/secrets/openshift-etcd/{new_secret_name}")), + serde_json::to_string(&etcd_value).context("serializing value")?.as_bytes().to_vec(), + ) + .await; + + etcd_client.delete(&key).await.context(format!("deleting {}", key))?; + + Ok(()) + }), + ) + .await + .into_iter() + .collect::>>()?; + } + + Ok(()) +} + +pub(crate) async fn fix_etcd_pod(etcd_client: &Arc, original_hostname: &str, hostname: &str) -> Result<()> { + join_all( + etcd_client + .list_keys("configmaps/openshift-etcd/etcd-pod") + .await? + .into_iter() + .chain( + etcd_client + .list_keys("configmaps/openshift-etcd/restore-etcd-pod") + .await? + .into_iter(), + ) + .map(|key| async move { + let etcd_result = etcd_client + .get(key.clone()) + .await + .with_context(|| format!("getting key {:?}", key))? + .context("key disappeared")?; + let value: Value = serde_yaml::from_slice(etcd_result.value.as_slice()) + .with_context(|| format!("deserializing value of key {:?}", key,))?; + let k8s_resource_location = K8sResourceLocation::try_from(&value)?; + + let mut configmap = get_etcd_json(etcd_client, &k8s_resource_location) + .await? + .context("getting secret")?; + + let data = &mut configmap + .pointer_mut("/data") + .context("no /data")? + .as_object_mut() + .context("data not an object")?; + + // TODO: We can't roundtrip arbitrary YAML, ask etcd folks to stop using YAML + // That's why we have to do primitive string manipulation here instead of proper + // parsing + let pod_yaml = data + .get_mut("pod.yaml") + .context("no pod.yaml")? + .as_str() + .context("pod.yaml not a string")? + .to_string(); + + let pod_yaml = fix_etcd_pod_yaml(&pod_yaml, original_hostname, hostname).context("could not fix pod yaml")?; + + data.insert("pod.yaml".to_string(), serde_json::Value::String(pod_yaml)); + + put_etcd_yaml(etcd_client, &k8s_resource_location, configmap).await?; + + Ok(()) + }), + ) + .await + .into_iter() + .collect::>>()?; + + Ok(()) +} + +pub(crate) async fn fix_etcd_scripts(etcd_client: &Arc, original_hostname: &str, hostname: &str) -> Result<()> { + let k8s_resource_location = K8sResourceLocation::new(Some("openshift-etcd"), "ConfigMap", "etcd-scripts", "v1"); + let mut configmap = get_etcd_json(etcd_client, &k8s_resource_location) + .await? + .context("getting configmap")?; + + let data = &mut configmap + .pointer_mut("/data") + .context("no /data")? + .as_object_mut() + .context("data not an object")?; + + // TODO: We can't roundtrip arbitrary YAML, ask etcd folks to stop using YAML + // That's why we have to do primitive string manipulation here instead of proper + // parsing + let mut pod_yaml = data + .get_mut("etcd.env") + .context("no etcd.env")? + .as_str() + .context("etcd.env not a string")? + .to_string(); + + let patterns = [ + ( + format!(r#"export NODE_{original_hostname}_ETCD_NAME="{original_hostname}""#), + r#"export NODE_{}_="{}""#, + ), + ( + format!(r#"export NODE_({original_hostname})_ETCD_URL_HOST="#), + r#"export NODE_{}_ETCD_URL_HOST="#, + ), + (format!(r#"export NODE_{original_hostname}_IP="#), r#"export NODE_{}_IP="#), + ]; + + for (pattern, replacement) in patterns { + let re = regex::Regex::new(&pattern).context("compiling regex")?; + pod_yaml = re + .replace_all(&pod_yaml, replacement.replace("{}", &env_var_safe(hostname)).as_str()) + .to_string(); + } + + data.insert("etcd.env".to_string(), serde_json::Value::String(pod_yaml)); + + put_etcd_yaml(etcd_client, &k8s_resource_location, configmap).await?; + + Ok(()) +} + +pub(crate) async fn fix_kubeapiservers_cluster(etcd_client: &Arc, hostname: &str) -> Result<()> { + let k8s_resource_location = K8sResourceLocation::new(None, "KubeAPIServer", "cluster", "operator.openshift.io/v1"); + let mut cluster = get_etcd_json(etcd_client, &k8s_resource_location) + .await? + .context("getting kubeapiservers/cluster")?; + + replace_node_status_name(&mut cluster, hostname).context("could not replace nodeName for kubeapiservers/cluster")?; + + put_etcd_yaml(etcd_client, &k8s_resource_location, cluster).await?; + + Ok(()) +} + +pub(crate) async fn fix_kubeschedulers_cluster(etcd_client: &Arc, hostname: &str) -> Result<()> { + let k8s_resource_location = K8sResourceLocation::new(None, "KubeScheduler", "cluster", "operator.openshift.io/v1"); + let mut cluster = get_etcd_json(etcd_client, &k8s_resource_location) + .await? + .context("getting kubeschedulers/cluster")?; + + replace_node_status_name(&mut cluster, hostname).context("could not replace nodeName for kubeschedulers/cluster")?; + + put_etcd_yaml(etcd_client, &k8s_resource_location, cluster).await?; + + Ok(()) +} + +pub(crate) async fn fix_kubecontrollermanagers_cluster(etcd_client: &Arc, hostname: &str) -> Result<()> { + let k8s_resource_location = K8sResourceLocation::new(None, "KubeControllerManager", "cluster", "operator.openshift.io/v1"); + let mut cluster = get_etcd_json(etcd_client, &k8s_resource_location) + .await? + .context("getting kubecontrollermanagers/cluster")?; + + replace_node_status_name(&mut cluster, hostname).context("could not replace nodeName for kubecontrollermanagers/cluster")?; + + put_etcd_yaml(etcd_client, &k8s_resource_location, cluster).await?; + + Ok(()) +} + +fn replace_node_status_name(cluster: &mut Value, hostname: &str) -> Result<()> { + let node_statuses = &mut cluster + .pointer_mut("/status/nodeStatuses") + .context("no /status/nodeStatuses")? + .as_array_mut() + .context("/status/nodeStatuses not an array")?; + + node_statuses.iter_mut().for_each(|status: &mut Value| { + status + .as_object_mut() + .unwrap() + .insert("nodeName".to_string(), Value::String(hostname.to_string())); + }); + + Ok(()) +} diff --git a/src/ocp_postprocess/hostname_rename/filesystem_rename.rs b/src/ocp_postprocess/hostname_rename/filesystem_rename.rs new file mode 100644 index 00000000..0e12512b --- /dev/null +++ b/src/ocp_postprocess/hostname_rename/filesystem_rename.rs @@ -0,0 +1,234 @@ +use crate::{ + file_utils::{self, commit_file, read_file_to_string}, + ocp_postprocess::cluster_domain_rename::rename_utils, +}; +use anyhow::{self, Context, Result}; +use futures_util::future::join_all; +use serde_json::Value; +use std::path::Path; + +pub(crate) async fn fix_filesystem_etcd_static_pods(original_hostname: &str, hostname: &str, dir: &Path) -> Result<()> { + join_all(file_utils::globvec(dir, "**/etcd-pod.yaml")?.into_iter().map(|file_path| { + let etcd_pod_path = file_path.clone(); + let original_hostname = original_hostname.to_string(); + let hostname = hostname.to_string(); + tokio::spawn(async move { + async move { + let contents = read_file_to_string(&file_path) + .await + .context("reading kube-controller-manager-pod.yaml")?; + + let mut pod: Value = serde_json::from_str(&contents).context("parsing etcd.yaml")?; + + rename_utils::fix_etcd_static_pod(&mut pod, &original_hostname, &hostname).context("fixing etcd-pod.yaml")?; + + commit_file(file_path, serde_json::to_string(&pod).context("serializing etcd-pod.yaml")?) + .await + .context("writing etcd-pod.yaml to disk")?; + + anyhow::Ok(()) + } + .await + .context(format!("fixing etcd-pod.yaml {:?}", etcd_pod_path)) + }) + })) + .await + .into_iter() + .collect::, _>>()? + .into_iter() + .collect::>>()?; + + Ok(()) +} + +pub(crate) async fn fix_filesystem_etcd_configmap_pod_yaml(original_hostname: &str, hostname: &str, dir: &Path) -> Result<()> { + join_all(file_utils::globvec(dir, "**/*etcd-pod/pod.yaml")?.into_iter().map(|file_path| { + let etcd_pod_path = file_path.clone(); + let original_hostname = original_hostname.to_string(); + let hostname = hostname.to_string(); + tokio::spawn(async move { + async move { + let contents = read_file_to_string(&file_path) + .await + .context("reading kube-controller-manager-pod.yaml")?; + + commit_file( + file_path, + rename_utils::fix_etcd_pod_yaml(&contents, &original_hostname, &hostname).context("fixing etcd-pod.yaml")?, + ) + .await + .context("writing etcd-pod.yaml to disk")?; + + anyhow::Ok(()) + } + .await + .context(format!("fixing etcd-pod.yaml {:?}", etcd_pod_path)) + }) + })) + .await + .into_iter() + .collect::, _>>()? + .into_iter() + .collect::>>()?; + + Ok(()) +} + +pub(crate) async fn fix_filesystem_etcd_scripts_cluster_backup_sh(original_hostname: &str, hostname: &str, dir: &Path) -> Result<()> { + join_all( + file_utils::globvec(dir, "**/etcd-scripts/cluster-backup.sh")? + .into_iter() + .map(|file_path| { + let cluster_backup_path = file_path.clone(); + let original_hostname = original_hostname.to_string(); + let hostname = hostname.to_string(); + tokio::spawn(async move { + async move { + let contents = read_file_to_string(&file_path).await.context("reading cluster-backup.sh")?; + + commit_file( + file_path, + rename_utils::fix_cluster_backup_sh(&contents, &original_hostname, &hostname) + .context("fixing cluster-backup.sh")?, + ) + .await + .context("writing cluster-backup.sh to disk")?; + + anyhow::Ok(()) + } + .await + .context(format!("fixing cluster-backup.sh {:?}", cluster_backup_path)) + }) + }), + ) + .await + .into_iter() + .collect::, _>>()? + .into_iter() + .collect::>>()?; + + Ok(()) +} + +pub(crate) async fn fix_filesystem_etcd_scripts_etcd_env(original_hostname: &str, hostname: &str, dir: &Path) -> Result<()> { + join_all(file_utils::globvec(dir, "**/etcd-scripts/etcd.env")?.into_iter().map(|file_path| { + let etcd_env_path = file_path.clone(); + let original_hostname = original_hostname.to_string(); + let hostname = hostname.to_string(); + tokio::spawn(async move { + async move { + let contents = read_file_to_string(&file_path).await.context("reading etcd.env")?; + + commit_file( + file_path, + rename_utils::fix_etcd_env(&contents, &original_hostname, &hostname).context("fixing etcd.env")?, + ) + .await + .context("writing etcd.env to disk")?; + + anyhow::Ok(()) + } + .await + .context(format!("fixing etcd.env {:?}", etcd_env_path)) + }) + })) + .await + .into_iter() + .collect::, _>>()? + .into_iter() + .collect::>>()?; + + Ok(()) +} + +pub(crate) async fn fix_filesystem_kapi_startup_monitor_pod(hostname: &str, dir: &Path) -> Result<()> { + join_all( + file_utils::globvec(dir, "**/kube-apiserver-startup-monitor-pod.yaml")? + .into_iter() + .map(|file_path| { + let kapi_startup_monitor_pod_path = file_path.clone(); + let hostname = hostname.to_string(); + tokio::spawn(async move { + async move { + let contents = read_file_to_string(&file_path) + .await + .context("reading kube-apiserver-startup-monitor-pod.yaml")?; + + let mut pod: Value = serde_json::from_str(&contents).context("parsing kube-apiserver-startup-monitor-pod.yaml")?; + + rename_utils::fix_kapi_startup_monitor_pod_container_args(&mut pod, &hostname) + .context("fixing kube-apiserver-startup-monitor-pod.yaml")?; + + commit_file( + file_path, + serde_json::to_string(&pod).context("serializing kube-apiserver-startup-monitor-pod.yaml")?, + ) + .await + .context("writing kube-apiserver-startup-monitor-pod.yaml to disk")?; + + anyhow::Ok(()) + } + .await + .context(format!( + "fixing kube-apiserver-startup-monitor-pod.yaml {:?}", + kapi_startup_monitor_pod_path + )) + }) + }), + ) + .await + .into_iter() + .collect::, _>>()? + .into_iter() + .collect::>>()?; + + Ok(()) +} + +pub(crate) async fn fix_filesystem_kapi_startup_monitor_configmap_pod_yaml( + original_hostname: &str, + hostname: &str, + dir: &Path, +) -> Result<()> { + join_all( + file_utils::globvec( + dir, + "**/kube-apiserver-pod*/configmaps/kube-apiserver-pod/kube-apiserver-startup-monitor-pod.yaml", + )? + .into_iter() + .map(|file_path| { + let kapi_startup_monitor_pod_path = file_path.clone(); + let original_hostname = original_hostname.to_string(); + let hostname = hostname.to_string(); + tokio::spawn(async move { + async move { + let contents = read_file_to_string(&file_path) + .await + .context("reading kube-apiserver-startup-monitor-pod.yaml")?; + + commit_file( + file_path, + rename_utils::fix_kapi_startup_monitor_pod_yaml(&contents, &original_hostname, &hostname) + .context("fixing kube-apiserver-startup-monitor-pod.yaml")?, + ) + .await + .context("writing kube-apiserver-startup-monitor-pod.yaml to disk")?; + + anyhow::Ok(()) + } + .await + .context(format!( + "fixing kube-apiserver-startup-monitor-pod.yaml {:?}", + kapi_startup_monitor_pod_path + )) + }) + }), + ) + .await + .into_iter() + .collect::, _>>()? + .into_iter() + .collect::>>()?; + + Ok(()) +} diff --git a/src/recert.rs b/src/recert.rs index 6cc0f69a..b6d31ca5 100644 --- a/src/recert.rs +++ b/src/recert.rs @@ -61,6 +61,7 @@ pub(crate) async fn run( Arc::clone(&in_memory_etcd_client), cluster_crypto, &parsed_cli.cluster_rename, + &parsed_cli.hostname, &parsed_cli.static_dirs, &parsed_cli.static_files, parsed_cli.regenerate_server_ssh_keys.as_deref(), @@ -117,10 +118,12 @@ async fn recertify( Ok((rsa_key_pool_and_scanning_run_time, processing_run_time)) } +#[allow(clippy::too_many_arguments)] async fn finalize( in_memory_etcd_client: Arc, cluster_crypto: &mut ClusterCryptoObjects, cluster_rename: &Option, + hostname: &Option, static_dirs: &Vec, static_files: &Vec, regenerate_server_ssh_keys: Option<&Path>, @@ -136,7 +139,7 @@ async fn finalize( let start = std::time::Instant::now(); if in_memory_etcd_client.etcd_client.is_some() { - ocp_postprocess(&in_memory_etcd_client, cluster_rename, static_dirs, static_files) + ocp_postprocess(&in_memory_etcd_client, cluster_rename, hostname, static_dirs, static_files) .await .context("performing ocp specific post-processing")?; }