From 0ed08a6758417c2bfd2c28b6247689d0044c602c Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Fri, 30 Sep 2022 20:28:02 +0200 Subject: [PATCH] chore: poc Signed-off-by: Antonio Murdaca --- Cargo.toml | 2 + .../greenboot-grub2-set-counter.service | 13 -- dist/systemd/system/greenboot-trigger.service | 21 ++ dist/systemd/system/greenboot.service | 10 +- greenboot.spec | 11 +- src/main.rs | 211 ++++++++---------- 6 files changed, 121 insertions(+), 147 deletions(-) delete mode 100644 dist/systemd/system/greenboot-grub2-set-counter.service create mode 100644 dist/systemd/system/greenboot-trigger.service diff --git a/Cargo.toml b/Cargo.toml index 7dc6200..37051a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,5 @@ config = "0.13" pretty_env_logger = "0.4" nix = "0.25.0" glob = "0.3.0" +serde = "1.0" +serde_json = "1.0" diff --git a/dist/systemd/system/greenboot-grub2-set-counter.service b/dist/systemd/system/greenboot-grub2-set-counter.service deleted file mode 100644 index 0b92a0d..0000000 --- a/dist/systemd/system/greenboot-grub2-set-counter.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Greenboot - set grub2 boot counter in preparation of upgrade -#DefaultDependencies=no -Before=ostree-finalize-staged.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/libexec/greenboot/greenboot set-counter -MountFlags=slave - -[Install] -RequiredBy=ostree-finalize-staged.service \ No newline at end of file diff --git a/dist/systemd/system/greenboot-trigger.service b/dist/systemd/system/greenboot-trigger.service new file mode 100644 index 0000000..3dfd0bf --- /dev/null +++ b/dist/systemd/system/greenboot-trigger.service @@ -0,0 +1,21 @@ +[Unit] +Description=Greenboot - TODO 2 +DefaultDependencies=no +Conflicts=shutdown.target +Before=shutdown.target + +Wants=local-fs.target +After=local-fs.target + +Before=multi-user.target systemd-update-done.service +ConditionNeedsUpdate=|/etc +ConditionNeedsUpdate=|/var + +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=/usr/libexec/greenboot/greenboot stamp +Restart=no + +[Install] +WantedBy=multi-user.target diff --git a/dist/systemd/system/greenboot.service b/dist/systemd/system/greenboot.service index 816e3b4..1d525d6 100644 --- a/dist/systemd/system/greenboot.service +++ b/dist/systemd/system/greenboot.service @@ -1,15 +1,13 @@ [Unit] -Description=Greenboot TODO -After=default.target +Description=Greenboot - TODO +After=multi-user.target Before=boot-complete.target -Conflicts=shutdown.target -Before=shutdown.target [Service] Type=oneshot RemainAfterExit=yes ExecStart=/usr/libexec/greenboot/greenboot check -MountFlags=slave +Restart=no [Install] -RequiredBy=boot-complete.target \ No newline at end of file +RequiredBy=boot-complete.target diff --git a/greenboot.spec b/greenboot.spec index 383ec78..1424bb2 100644 --- a/greenboot.spec +++ b/greenboot.spec @@ -35,7 +35,6 @@ BuildRequires: rust-packaging BuildRequires: systemd-rpm-macros %{?systemd_requires} Requires: systemd >= 240 -Requires: grub2-tools-minimal Requires: rpm-ostree # PAM is required to programmatically read motd messages from /etc/motd.d/* # This causes issues with RHEL-8 as the fix isn't there an el8 is on pam-1.3.x @@ -53,7 +52,7 @@ Obsoletes: greenboot-status <= 0.12.0 Provides: greenboot-rpm-ostree-grub2 Obsoletes: greenboot-rpm-ostree-grub2 <= 0.12.0 # List of bundled crate in vendor tarball, generated with: -# cargo metadata --locked --format-version 1 | CRATE_NAME="greenboot" ../bundled-provides.jq +# cargo metadata --locked --format-version 1 | CRATE_NAME="greenboot" ./bundled-provides.jq Provides: bundled(crate(ahash)) = 0.7.6 Provides: bundled(crate(aho-corasick)) = 0.7.19 Provides: bundled(crate(anyhow)) = 1.0.65 @@ -197,15 +196,15 @@ mkdir -p %{buildroot}%{_tmpfilesdir} %post %systemd_post greenboot.service -%systemd_post greenboot-grub2-set-counter.service +%systemd_post greenboot-trigger.service %preun %systemd_preun greenboot.service -%systemd_preun greenboot-grub2-set-counter.service +%systemd_preun greenboot-trigger.service %postun %systemd_postun greenboot.service -%systemd_postun greenboot-grub2-set-counter.service +%systemd_postun greenboot-trigger.service %files %doc README.md @@ -213,7 +212,7 @@ mkdir -p %{buildroot}%{_tmpfilesdir} %dir %{_libexecdir}/%{name} %{_libexecdir}/%{name}/%{name} %{_unitdir}/greenboot.service -%{_unitdir}/greenboot-grub2-set-counter.service +%{_unitdir}/greenboot-trigger.service %dir %{_prefix}/lib/%{name} %dir %{_prefix}/lib/%{name}/check %dir %{_prefix}/lib/%{name}/check/required.d diff --git a/src/main.rs b/src/main.rs index 9c664c8..a55df5b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,16 @@ -use std::{collections::HashMap, io::Write, os::unix::prelude::AsRawFd, process::Command}; +use std::hash::Hash; +use std::io::ErrorKind; +use std::iter::FromIterator; +use std::{ + collections::HashSet, + fs::{self, File}, + process::Command, +}; use anyhow::{bail, Error, Result}; -use clap::{Args, Parser, Subcommand, ValueEnum}; +use clap::{Parser, Subcommand, ValueEnum}; use glob::glob; -use nix::mount::{mount, MsFlags}; +use serde::{Deserialize, Serialize}; #[derive(Parser)] #[clap(author, version, about, long_about = None)] @@ -41,35 +48,27 @@ impl LogLevel { #[derive(Subcommand)] enum Commands { - Check(CheckArguments), - SetCounter(SetCounterArguments), + Check, + Stamp, } -#[derive(Args)] -struct CheckArguments {} - -#[derive(Args)] -struct SetCounterArguments {} - -fn check(_args: &CheckArguments) -> Result<(), Error> { - // TODO: run only if boot_success=0 && boot_counter != "" || empty too - - // TODO: logic for watchdog - if is_boot_wd_triggered()? { - // do something for wd triggered boot - } +#[derive(Serialize, Deserialize, PartialEq, Eq, Hash)] +struct ServiceStatus { + unit: String, +} - let grub2_editenv_list = parse_grub2_editenv_list()?; - if let Some(v) = grub2_editenv_list.get("boot_counter") { - if v == "-1" { - // TODO: cleanup "bad" upgrade deployment, there's a command I don't remember... - Command::new("rpm-ostree").arg("rollback").status()?; - Command::new("grub2-editenv") - .arg("-") - .arg("unset") - .arg("boot_counter") - .status()?; +fn check() -> Result<(), Error> { + match File::open("/etc/greenboot/upgrade.stamp") { + Ok(_) => { + log::info!("stamp on disk, removing and running greenboot"); + std::fs::remove_file("/etc/greenboot/upgrade.stamp")? } + Err(e) => match e.kind() { + ErrorKind::NotFound => return Ok(()), + _ => { + bail!("unknown error when opening stamp file: {:?}", e); + } + }, } let mut failure = false; for path in [ @@ -77,49 +76,73 @@ fn check(_args: &CheckArguments) -> Result<(), Error> { "/etc/greenboot/check/required.d/*.sh", ] { for entry in glob(path)?.flatten() { - let status = Command::new("bash").arg("-C").arg(entry).status()?; - if !status.success() { + log::info!("running required check {}", entry.to_string_lossy()); + let output = Command::new("bash").arg("-C").arg(entry).output()?; + if !output.status.success() { + // combine and print stderr/stdout log::warn!("required script failed..."); failure = true; } } } - // for path in [ - // "/usr/lib/greenboot/check/wanted.d/*.sh", - // "/etc/greenboot/check/wanted.d/*.sh", - // ] { - // for entry in glob(path)?.flatten() { - // let status = Command::new("bash").arg("-C").arg(entry).status()?; - // if !status.success() { - // log::warn!("wanted script failed..."); - // } - // } - // } - if failure { - // TODO: run red checks... - log::warn!("required scripts failed, check logs, exiting..."); - if !grub2_editenv_list.contains_key("boot_counter") { - bail!("<0>SYSTEM is UNHEALTHY, but boot_counter is unset in grubenv. Manual intervention necessary."); + for path in [ + "/usr/lib/greenboot/check/wanted.d/*.sh", + "/etc/greenboot/check/wanted.d/*.sh", + ] { + for entry in glob(path)?.flatten() { + log::info!("running required check {}", entry.to_string_lossy()); + let output = Command::new("bash").arg("-C").arg(entry).output()?; + if !output.status.success() { + // combine and print stderr/stdout + log::warn!("wanted script failed..."); + } + } + } + // if a command with restart option in systemd fails to start we don't get it as "failed" + // reversing the check makes sure that if by the time After=multi-user the service isn't running then it's failing at least + let output = Command::new("systemctl") + .arg("list-units") + .arg("--state") + .arg("active") + .arg("--no-page") + .arg("--output") + .arg("json") + .output()?; + let services: Vec = serde_json::from_str(&String::from_utf8(output.stdout)?)?; + let ss: Vec = services.iter().map(|x| x.unit.clone()).collect(); + let active_units: HashSet = HashSet::from_iter(ss); + for service in ["sshd.service", "NetworkManager.service"] { + if !active_units.contains(service) { + log::warn!("service {} failed, see journal", service); + failure = true; } - if glob("/boot/loader/entries/*")?.count() == 1 { - bail!("<0>SYSTEM is UNHEALTHY, but bootlader entry count is 1. Manual intervention necessary."); + } + if failure { + for path in ["/etc/greenboot/red.d/*.sh"] { + for entry in glob(path)?.flatten() { + log::info!("running red check {}", entry.to_string_lossy()); + let output = Command::new("bash").arg("-C").arg(entry).output()?; + if !output.status.success() { + // combine and print stderr/stdout + log::warn!("red script failed..."); + } + } } - log::warn!("<1>SYSTEM is UNHEALTHY. Rebooting..."); + log::warn!("SYSTEM is UNHEALTHY. Rolling back and rebooting..."); + Command::new("rpm-ostree").arg("rollback").status()?; reboot()?; return Ok(()); } - // TODO: run green checks... - // TODO: if we are here, we need to cleanup all other previous deployments - Command::new("grub2-editenv") - .arg("-") - .arg("set") - .arg("boot_success=1") - .status()?; - Command::new("grub2-editenv") - .arg("-") - .arg("unset") - .arg("boot_counter") - .status()?; + for path in ["/etc/greenboot/green.d/*.sh"] { + for entry in glob(path)?.flatten() { + log::info!("running green check {}", entry.to_string_lossy()); + let output = Command::new("bash").arg("-C").arg(entry).output()?; + if !output.status.success() { + // combine and print stderr/stdout + log::warn!("green script failed..."); + } + } + } Ok(()) } @@ -128,66 +151,12 @@ fn reboot() -> Result<(), Error> { Ok(()) } -const WATCHDOG_IOCTL_BASE: u8 = b'W'; -const WDIOC_TYPE_MODE: u8 = 2; -nix::ioctl_read!(wd_getbootstatus, WATCHDOG_IOCTL_BASE, WDIOC_TYPE_MODE, i32); - -fn from_nix_result(res: ::nix::Result) -> std::io::Result { - match res { - Ok(r) => Ok(r), - Err(err) => Err(err.into()), - } -} - -fn is_boot_wd_triggered() -> Result { - let mut devfile = std::fs::OpenOptions::new(); - devfile.read(true).write(true).create(false); - let mut wd = match devfile.open("/dev/watchdog") { - Ok(file) => file, - Err(_) => { - log::warn!("no watchdog"); - return Ok(false); - } - }; - let mut boot_status: i32 = 0; - from_nix_result(unsafe { wd_getbootstatus(wd.as_raw_fd(), &mut boot_status) })?; - wd.write_all("V".as_bytes())?; - Ok(boot_status == 1) -} - -fn set_counter(_args: &SetCounterArguments) -> Result<()> { - // all commands for grub2/systemctl need an abstraction to mock them in testing... - Command::new("grub2-editenv") - .arg("-") - .arg("set") - .arg("boot_success=0") - .spawn()?; - Command::new("grub2-editenv") - .arg("-") - .arg("set") - .arg("boot_counter=1") - .spawn()?; +fn stamp() -> Result<(), Error> { + fs::create_dir_all("/etc/greenboot/")?; + File::create("/etc/greenboot/upgrade.stamp")?; Ok(()) } -fn parse_grub2_editenv_list() -> Result> { - let output = Command::new("grub2-editenv").arg("list").output()?; - let stdout = String::from_utf8(output.stdout)?; - let split = stdout.split('\n').collect::>(); - let mut hm = HashMap::new(); - for s in split { - if s.is_empty() { - continue; - } - let ss = s.split('=').collect::>(); - if ss.len() != 2 { - continue; - } - hm.insert(ss[0].to_string(), ss[1].to_string()); - } - Ok(hm) -} - fn main() -> Result<()> { let cli = Cli::parse(); @@ -195,10 +164,8 @@ fn main() -> Result<()> { .filter_level(cli.log_level.to_log()) .init(); - mount::(None, "/boot", None, MsFlags::MS_REMOUNT, None)?; - match cli.command { - Commands::Check(args) => check(&args), - Commands::SetCounter(args) => set_counter(&args), + Commands::Check => check(), + Commands::Stamp => stamp(), } }