From 65cf3102c0cfa66f85411e837e0a7276eef96628 Mon Sep 17 00:00:00 2001 From: jlanson Date: Fri, 6 Dec 2024 14:07:40 -0500 Subject: [PATCH] feat: add `manifest` xtask to parse GitHub Release API output This commit is an initial part of a larger effort to develop an `xtask` command to automate plugin download manifest file updates when new plugin releases are made. This commit simply calls the GitHub release API for hipcheck, and parses and manipulates the returned JSON into DownloadManifestEntry objects that can be compared against the entries in existing download manifest files. Signed-off-by: jlanson --- Cargo.lock | 17 +- xtask/Cargo.toml | 18 + xtask/src/main.rs | 4 + xtask/src/task/manifest/download_manifest.rs | 541 ++++++++++++++++++ xtask/src/task/manifest/kdl.rs | 95 +++ xtask/src/task/manifest/mod.rs | 19 + xtask/src/task/manifest/release.rs | 2 + xtask/src/task/manifest/remote.rs | 130 +++++ xtask/src/task/manifest/util/agent.rs | 32 ++ .../task/manifest/util/authenticated_agent.rs | 54 ++ xtask/src/task/manifest/util/mod.rs | 5 + xtask/src/task/manifest/util/redacted.rs | 33 ++ xtask/src/task/mod.rs | 1 + 13 files changed, 946 insertions(+), 5 deletions(-) create mode 100644 xtask/src/task/manifest/download_manifest.rs create mode 100644 xtask/src/task/manifest/kdl.rs create mode 100644 xtask/src/task/manifest/mod.rs create mode 100644 xtask/src/task/manifest/release.rs create mode 100644 xtask/src/task/manifest/remote.rs create mode 100644 xtask/src/task/manifest/util/agent.rs create mode 100644 xtask/src/task/manifest/util/authenticated_agent.rs create mode 100644 xtask/src/task/manifest/util/mod.rs create mode 100644 xtask/src/task/manifest/util/redacted.rs diff --git a/Cargo.lock b/Cargo.lock index d3703650..891cb9f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2874,21 +2874,21 @@ dependencies = [ [[package]] name = "miette" -version = "5.10.0" +version = "7.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +checksum = "317f146e2eb7021892722af37cf1b971f0a70c8406f487e24952667616192c64" dependencies = [ + "cfg-if", "miette-derive", - "once_cell", "thiserror 1.0.69", "unicode-width 0.1.11", ] [[package]] name = "miette-derive" -version = "5.10.0" +version = "7.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +checksum = "23c9b935fbe1d6cbd1dac857b54a688145e2d93f48db36010514d0f612d0ad67" dependencies = [ "proc-macro2", "quote", @@ -5152,10 +5152,17 @@ dependencies = [ "convert_case", "env_logger", "glob", + "kdl", "log", "pathbuf", + "regex", + "rustls", + "rustls-native-certs", "serde", + "serde_json", "toml", + "ureq", + "url", "which", "xshell", ] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 2fa2edf1..9031d5ad 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -22,3 +22,21 @@ toml = "0.8.19" xshell = "0.2.7" which = "7.0.0" convert_case = "0.6.0" +serde_json = "1.0.133" +url = { version = "2.5.4", features = ["serde"] } +kdl = "4.7.0" +regex = "1.11.1" +ureq = { version = "2.10.1", default-features = false, features = [ + "json", + "tls", +] } +# Exactly matching the version of rustls used by ureq +# Get rid of default features since we don't use the AWS backed crypto +# provider (we use ring) and it breaks stuff on windows. +rustls = { version = "0.23.10", default-features = false, features = [ + "logging", + "std", + "tls12", + "ring", +] } +rustls-native-certs = "0.8.0" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2db9ec95..daf78f1b 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -27,6 +27,7 @@ fn main() -> ExitCode { Commands::Site(args) => match args.command { SiteCommand::Serve(args) => task::site::serve::run(args), }, + Commands::Manifest => task::manifest::run(), }; match result { @@ -63,6 +64,9 @@ enum Commands { Site(SiteArgs), /// Lint the Hipcheck plugin gRPC definition. Buf, + /// Update the plugin download manifests in the local repo based on + /// output from the GitHub Releases API + Manifest, } #[derive(Debug, clap::Args)] diff --git a/xtask/src/task/manifest/download_manifest.rs b/xtask/src/task/manifest/download_manifest.rs new file mode 100644 index 00000000..3c577f7a --- /dev/null +++ b/xtask/src/task/manifest/download_manifest.rs @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + string_newtype_parse_kdl_node, + task::manifest::{ + kdl::{extract_data, ParseKdlNode, ToKdlNode}, + util::agent, + }, +}; +use anyhow::anyhow; +use kdl::{KdlDocument, KdlNode, KdlValue}; +use regex::Regex; +use std::{fmt::Display, str::FromStr, sync::LazyLock}; + +static VERSION_REGEX: LazyLock = + LazyLock::new(|| Regex::new("[v]?([0-9]+).([0-9]+).([0-9]+)").unwrap()); + +pub fn parse_plugin_version(version_str: &str) -> Option { + VERSION_REGEX.captures(version_str).map(|m| { + PluginVersion( + m.get(1).unwrap().as_str().parse::().unwrap(), + m.get(2).unwrap().as_str().parse::().unwrap(), + m.get(3).unwrap().as_str().parse::().unwrap(), + ) + }) +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct PluginVersion(pub u8, pub u8, pub u8); + +impl ParseKdlNode for PluginVersion { + fn kdl_key() -> &'static str { + "version" + } + + fn parse_node(node: &KdlNode) -> Option { + let raw_version = node.entries().first()?.value().as_string()?; + parse_plugin_version(raw_version) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Arch(pub String); +string_newtype_parse_kdl_node!(Arch, "arch"); + +// NOTE: the implementation in this crate was largely derived from RFD #0004 + +impl ParseKdlNode for url::Url { + fn kdl_key() -> &'static str { + "url" + } + + fn parse_node(node: &KdlNode) -> Option { + let raw_url = node.entries().first()?.value().as_string()?; + url::Url::from_str(raw_url).ok() + } +} + +/// Contains all of the hash algorithms supported inside of the plugin download manifest +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum HashAlgorithm { + Sha256, + Blake3, +} + +impl Display for HashAlgorithm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + HashAlgorithm::Sha256 => write!(f, "SHA256"), + HashAlgorithm::Blake3 => write!(f, "BLAKE3"), + } + } +} + +impl TryFrom<&str> for HashAlgorithm { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result { + match value { + "SHA256" => Ok(HashAlgorithm::Sha256), + "BLAKE3" => Ok(HashAlgorithm::Blake3), + _ => Err(anyhow!("Invalid hash algorithm specified: '{}'", value)), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum BufferedDigest { + Resolved(String), + Remote(url::Url), +} +impl BufferedDigest { + #[allow(unused)] // Will be used in a later PR + fn resolve(self) -> anyhow::Result { + use BufferedDigest::*; + match self { + Resolved(s) => Ok(s), + Remote(u) => { + let agent = agent::agent(); + let raw_hash_str = agent.get(u.as_ref()).call()?.into_string()?; + let Some(hash_str) = raw_hash_str.split_whitespace().next() else { + return Err(anyhow!("malformed sha256 file at {}", u)); + }; + Ok(hash_str.to_owned()) + } + } + } +} +impl From for BufferedDigest { + fn from(value: String) -> Self { + BufferedDigest::Resolved(value) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HashWithDigest { + /// hash algorithm used + pub hash_algorithm: HashAlgorithm, + /// expected hash of artifact when hashed with the hash_algorithm specified + pub digest: BufferedDigest, +} + +impl HashWithDigest { + pub fn new(hash_algorithm: HashAlgorithm, digest: BufferedDigest) -> Self { + Self { + hash_algorithm, + digest, + } + } +} + +impl ParseKdlNode for HashWithDigest { + fn kdl_key() -> &'static str { + "hash" + } + + fn parse_node(node: &KdlNode) -> Option { + if node.name().to_string().as_str() != Self::kdl_key() { + return None; + } + // Per RFD #0004, the hash algorithm is of type String + let specified_algorithm = node.get("alg")?.value().as_string()?; + let hash_algorithm = HashAlgorithm::try_from(specified_algorithm).ok()?; + // Per RFD #0004, the digest is of type String + let digest = node.get("digest")?.value().as_string()?.to_string(); + Some(HashWithDigest::new( + hash_algorithm, + BufferedDigest::Resolved(digest), + )) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ArchiveFormat { + /// archived with tar and compressed with the XZ algorithm + TarXz, + /// archived with tar and compressed with the Gzip algorithm + TarGz, + /// archived with tar and compressed with the zstd algorithm + TarZst, + /// archived with tar, not compressed + Tar, + /// archived and compressed with zip + Zip, +} + +impl Display for ArchiveFormat { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ArchiveFormat::TarXz => write!(f, "tar.xz"), + ArchiveFormat::TarGz => write!(f, "tar.gz"), + ArchiveFormat::TarZst => write!(f, "tar.zst"), + ArchiveFormat::Tar => write!(f, "tar"), + ArchiveFormat::Zip => write!(f, "zip"), + } + } +} + +impl TryFrom<&str> for ArchiveFormat { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result { + match value { + "tar.xz" => Ok(ArchiveFormat::TarXz), + "tar.gz" => Ok(ArchiveFormat::TarGz), + "tar.zst" => Ok(ArchiveFormat::TarZst), + "tar" => Ok(ArchiveFormat::Tar), + "zip" => Ok(ArchiveFormat::Zip), + _ => Err(anyhow!("Invalid compression format specified")), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Compress { + /// compression algorithm used for the downloaded archive + pub format: ArchiveFormat, +} + +impl Compress { + #[cfg(test)] + pub fn new(archive_format: ArchiveFormat) -> Self { + Self { + format: archive_format, + } + } +} + +impl ParseKdlNode for Compress { + fn kdl_key() -> &'static str { + "compress" + } + + fn parse_node(node: &KdlNode) -> Option { + if node.name().to_string().as_str() != Self::kdl_key() { + return None; + } + // Per RFD #0004, the format is of type String + let specified_format = node.get("format")?.value().as_string()?; + let format = ArchiveFormat::try_from(specified_format).ok()?; + Some(Compress { format }) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Size { + /// size of the downloaded artifact, in bytes + pub bytes: u64, +} + +impl Size { + pub fn new(bytes: u64) -> Self { + Self { bytes } + } +} + +impl ParseKdlNode for Size { + fn kdl_key() -> &'static str { + "size" + } + + fn parse_node(node: &KdlNode) -> Option { + if node.name().to_string().as_str() != Self::kdl_key() { + return None; + } + let specified_size = node.get("bytes")?; + let bytes = match specified_size.value() { + // Negative size and a size of 0 do not make sense + KdlValue::Base10(bytes) => { + let bytes = *bytes; + if bytes.is_positive() { + bytes as u64 + } else { + return None; + } + } + _ => return None, + }; + Some(Size { bytes }) + } +} + +/// Represents one entry in a download manifest file, as spelled out in RFD #0004 +/// Example entry: +/// ``` +///plugin version="0.1.0" arch="aarch64-apple-darwin" { +/// url "https://github.com/mitre/hipcheck/releases/download/hipcheck-v3.4.0/hipcheck-aarch64-apple-darwin.tar.xz" +/// hash alg="SHA256" digest="b8e111e7817c4a1eb40ed50712d04e15b369546c4748be1aa8893b553f4e756b" +/// compress format="tar.xz" +/// size bytes=2_869_896 +///} +///``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DownloadManifestEntry { + // TODO: make this a SemVer type? + /// A `SemVer` version of the plugin. Not a version requirement as in the plugin manifest file, + /// but only a specific concrete version + pub version: PluginVersion, + /// The target architecture for a plugin + pub arch: Arch, + /// The URL of the archive file to download containing the plugin executable artifact and + /// plugin manifest. + pub url: url::Url, + /// Contains info about what algorithm was used to hash the archive and what the expected + /// digest is + pub hash: HashWithDigest, + /// Defines how to handle decompressing the downloaded plugin archive + pub compress: Compress, + /// Describes the size of the downloaded artifact, used to validate the download was + /// successful, makes it more difficult for an attacker to distribute malformed artifacts + pub size: Size, +} + +impl ParseKdlNode for DownloadManifestEntry { + fn kdl_key() -> &'static str { + "plugin" + } + + fn parse_node(node: &KdlNode) -> Option { + if node.name().to_string().as_str() != Self::kdl_key() { + return None; + } + // Per RFD #0004, arch is of type String + let arch = Arch(node.get("arch")?.value().as_string()?.to_string()); + let version = parse_plugin_version(node.get("version")?.value().as_string()?)?; + + // there should be one child for each plugin and it should contain the url, hash, compress + + // and size information + let nodes = node.children()?.nodes(); + + // extract the url, hash, compress and size from the child + let url: url::Url = extract_data(nodes)?; + let hash: HashWithDigest = extract_data(nodes)?; + let compress: Compress = extract_data(nodes)?; + let size: Size = extract_data(nodes)?; + + Some(Self { + version, + arch, + url, + hash, + compress, + size, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DownloadManifest { + pub entries: Vec, +} + +impl DownloadManifest { + #[cfg(test)] + pub fn iter(&self) -> impl Iterator { + self.entries.iter() + } + + #[cfg(test)] + pub fn len(&self) -> usize { + self.entries.len() + } +} + +impl FromStr for DownloadManifest { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let document = KdlDocument::from_str(s) + .map_err(|e| anyhow!("Error parsing download manifest file: {}", e.to_string()))?; + let mut entries = vec![]; + for node in document.nodes() { + if let Some(entry) = DownloadManifestEntry::parse_node(node) { + entries.push(entry); + } else { + return Err(anyhow!("Error parsing download manifest entry: {}", node)); + } + } + Ok(Self { entries }) + } +} + +#[cfg(test)] +mod test { + use super::*; + use std::str::FromStr; + use url::Url; + + #[test] + fn test_parsing_hash_algorithm() { + let digest = "b8e111e7817c4a1eb40ed50712d04e15b369546c4748be1aa8893b553f4e756b"; + + // test SHA256 parsing + let node = + KdlNode::from_str(format!(r#" hash alg="SHA256" digest="{}""#, digest).as_str()) + .unwrap(); + let parsed_hash_with_digest = HashWithDigest::parse_node(&node).unwrap(); + let expected_hash_with_digest = + HashWithDigest::new(HashAlgorithm::Sha256, digest.to_string().into()); + assert_eq!(parsed_hash_with_digest, expected_hash_with_digest); + + // test BLAKE3 parsing + let node = + KdlNode::from_str(format!(r#" hash alg="BLAKE3" digest="{}""#, digest).as_str()) + .unwrap(); + let parsed_hash_with_digest = HashWithDigest::parse_node(&node).unwrap(); + let expected_hash_with_digest = + HashWithDigest::new(HashAlgorithm::Blake3, digest.to_string().into()); + assert_eq!(parsed_hash_with_digest, expected_hash_with_digest); + + // ensure invalid hash algorithms do not pass + let node = KdlNode::from_str(format!(r#" hash alg="SHA1" digest="{}""#, digest).as_str()) + .unwrap(); + assert!(HashWithDigest::parse_node(&node).is_none()); + let node = KdlNode::from_str(format!(r#" hash alg="BLAKE" digest="{}""#, digest).as_str()) + .unwrap(); + assert!(HashWithDigest::parse_node(&node).is_none()); + } + + #[test] + fn test_parsing_compression_algorithm() { + let formats = [ + ("tar.xz", ArchiveFormat::TarXz), + ("tar.gz", ArchiveFormat::TarGz), + ("tar.zst", ArchiveFormat::TarZst), + ("tar", ArchiveFormat::Tar), + ("zip", ArchiveFormat::Zip), + ]; + for (value, format) in formats { + let node = + KdlNode::from_str(format!(r#"compress format="{}""#, value).as_str()).unwrap(); + assert_eq!(Compress::parse_node(&node).unwrap(), Compress { format }); + } + } + + #[test] + fn test_parsing_size() { + // test normal number + let node = KdlNode::from_str("size bytes=1234").unwrap(); + assert_eq!(Size::parse_node(&node).unwrap(), Size { bytes: 1234 }); + let node = KdlNode::from_str("size bytes=1").unwrap(); + assert_eq!(Size::parse_node(&node).unwrap(), Size { bytes: 1 }); + + // test parsing number with _ inside + let node = KdlNode::from_str("size bytes=1_234_567").unwrap(); + assert_eq!(Size::parse_node(&node).unwrap(), Size { bytes: 1234567 }); + + // test that negative number does not work + let node = KdlNode::from_str("size bytes=-1234").unwrap(); + assert!(Size::parse_node(&node).is_none()); + + // ensure 0 does not work + let node = KdlNode::from_str("size bytes=0").unwrap(); + assert!(Size::parse_node(&node).is_none()); + + // ensure negative numbers do not work + let node = KdlNode::from_str("size bytes=-1").unwrap(); + assert!(Size::parse_node(&node).is_none()); + } + + #[test] + fn test_parsing_url() { + let raw_url = "https://github.com/mitre/hipcheck/releases/download/hipcheck-v3.4.0/hipcheck-x86_64-apple-darwin.tar.xz"; + let node = KdlNode::from_str(format!(r#"url "{}""#, raw_url).as_str()).unwrap(); + assert_eq!( + Url::parse_node(&node).unwrap(), + Url::parse(raw_url).unwrap() + ); + } + + #[test] + fn test_parsing_single_download_manifest_entry() { + let version = "0.1.0"; + let arch = "aarch64-apple-darwin"; + let url = "https://github.com/mitre/hipcheck/releases/download/hipcheck-v3.4.0/hipcheck-aarch64-apple-darwin.tar.xz"; + let hash_alg = "SHA256"; + let digest = "b8e111e7817c4a1eb40ed50712d04e15b369546c4748be1aa8893b553f4e756b"; + let compress = "tar.gz"; + let size = "2_869_896"; + + let node = KdlNode::from_str( + format!( + r#"plugin version="{}" arch="{}" {{ + url "{}" + hash alg="{}" digest="{}" + compress format="{}" + size bytes={} +}}"#, + version, arch, url, hash_alg, digest, compress, size + ) + .as_str(), + ) + .unwrap(); + + let expected_entry = DownloadManifestEntry { + version: parse_plugin_version(version).unwrap(), + arch: Arch(arch.to_string()), + url: Url::parse(url).unwrap(), + hash: HashWithDigest::new( + HashAlgorithm::try_from(hash_alg).unwrap(), + digest.to_string().into(), + ), + compress: Compress { + format: ArchiveFormat::try_from(compress).unwrap(), + }, + size: Size { + bytes: u64::from_str(size.replace("_", "").as_str()).unwrap(), + }, + }; + + assert_eq!( + DownloadManifestEntry::parse_node(&node).unwrap(), + expected_entry + ); + } + + #[test] + fn test_parsing_multiple_download_manifest_entry() { + let contents = r#"plugin version="0.1.0" arch="aarch64-apple-darwin" { + url "https://github.com/mitre/hipcheck/releases/download/hipcheck-v3.4.0/hipcheck-aarch64-apple-darwin.tar.xz" + hash alg="SHA256" digest="b8e111e7817c4a1eb40ed50712d04e15b369546c4748be1aa8893b553f4e756b" + compress format="tar.xz" + size bytes=2_869_896 +} + +plugin version="0.1.0" arch="x86_64-apple-darwin" { + url "https://github.com/mitre/hipcheck/releases/download/hipcheck-v3.4.0/hipcheck-x86_64-apple-darwin.tar.xz" + hash alg="SHA256" digest="ddb8c6d26dd9a91e11c99b3bd7ee2b9585aedac6e6df614190f1ba2bfe86dc19" + compress format="tar.xz" + size bytes=3_183_768 +}"#; + let entries = DownloadManifest::from_str(contents).unwrap(); + assert_eq!(entries.len(), 2); + let mut entries_iter = entries.iter(); + assert_eq!( + &DownloadManifestEntry { + version: PluginVersion(0, 1, 0), + arch: Arch("aarch64-apple-darwin".to_owned()), + url: Url::parse("https://github.com/mitre/hipcheck/releases/download/hipcheck-v3.4.0/hipcheck-aarch64-apple-darwin.tar.xz").unwrap(), + hash: HashWithDigest::new(HashAlgorithm::Sha256, "b8e111e7817c4a1eb40ed50712d04e15b369546c4748be1aa8893b553f4e756b".to_owned().into()), + compress: Compress::new(ArchiveFormat::TarXz), + size: Size { + bytes: 2_869_896 + } + }, + entries_iter.next().unwrap() + ); + assert_eq!( + &DownloadManifestEntry { + version: PluginVersion(0, 1, 0), + arch: Arch("x86_64-apple-darwin".to_owned()), + url: Url::parse("https://github.com/mitre/hipcheck/releases/download/hipcheck-v3.4.0/hipcheck-x86_64-apple-darwin.tar.xz").unwrap(), + hash: HashWithDigest::new(HashAlgorithm::Sha256, "ddb8c6d26dd9a91e11c99b3bd7ee2b9585aedac6e6df614190f1ba2bfe86dc19".to_owned().into()), + compress: Compress::new(ArchiveFormat::TarXz), + size: Size::new(3_183_768) + }, + entries_iter.next().unwrap() + ); + } +} diff --git a/xtask/src/task/manifest/kdl.rs b/xtask/src/task/manifest/kdl.rs new file mode 100644 index 00000000..bc785b49 --- /dev/null +++ b/xtask/src/task/manifest/kdl.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! General shared types and functions for KDL files + +use kdl::KdlNode; + +// Helper trait to make it easier to parse KdlNodes into our own types +pub trait ParseKdlNode +where + Self: Sized, +{ + /// Return the name of the attribute used to identify the node pertaining to this struct + fn kdl_key() -> &'static str; + + /// Attempt to convert a `kdl::KdlNode` into Self + fn parse_node(node: &KdlNode) -> Option; +} + +pub trait ToKdlNode { + /// convert self to a KdlNode + #[allow(unused)] + fn to_kdl_node(&self) -> KdlNode; +} + +/// Returns the first successful node that can be parsed into T, if there is one +pub fn extract_data(nodes: &[KdlNode]) -> Option +where + T: ParseKdlNode, +{ + for node in nodes { + if let Some(val) = T::parse_node(node) { + return Some(val); + } + } + None +} + +/// Use this macro to generate the code needed to parse a KDL node that is a single string, as the +/// code is quite repetitive for this simple task. +/// +/// As a bonus, the following code is also generated: +/// - `AsRef` +/// - `new(value: String) -> Self` +/// - `ToKdlNode` +/// +/// NOTE: This only works with newtype wrappers around String! +/// +/// Example: +/// publisher "mitre" can be generated by this macro! +/// +/// ```rust +/// struct Publisher(pub String) +/// ``` +#[macro_export] +macro_rules! string_newtype_parse_kdl_node { + ($type:ty, $identifier:expr) => { + impl $type { + #[allow(dead_code)] + pub fn new(value: String) -> Self { + Self(value) + } + } + + impl ParseKdlNode for $type { + fn kdl_key() -> &'static str { + $identifier + } + + fn parse_node(node: &KdlNode) -> Option { + if node.name().to_string().as_str() != Self::kdl_key() { + return None; + } + // NOTE: this macro currently assumes that the first positional argument is the + // correct value to be parsing, which is true for newtype String wrappers! + let entry = node.entries().first()?.value().as_string()?.to_string(); + Some(Self(entry)) + } + } + + impl AsRef for $type { + fn as_ref(&self) -> &String { + &self.0 + } + } + + impl ToKdlNode for $type { + #[allow(unused)] + fn to_kdl_node(&self) -> KdlNode { + let mut node = KdlNode::new(Self::kdl_key()); + node.insert(0, self.0.clone()); + node + } + } + }; +} diff --git a/xtask/src/task/manifest/mod.rs b/xtask/src/task/manifest/mod.rs new file mode 100644 index 00000000..daff8de8 --- /dev/null +++ b/xtask/src/task/manifest/mod.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 + +pub mod download_manifest; +mod kdl; +pub mod remote; +mod util; + +use anyhow::Result; + +pub fn run() -> Result<()> { + let api_token = std::env::var("HC_GITHUB_TOKEN")?; + let releases = remote::get_hipcheck_releases(&api_token)?; + + for (name, release) in releases.0 { + println!("{name}: {release:?}"); + } + + Ok(()) +} diff --git a/xtask/src/task/manifest/release.rs b/xtask/src/task/manifest/release.rs new file mode 100644 index 00000000..fda8e861 --- /dev/null +++ b/xtask/src/task/manifest/release.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: Apache-2.0 + diff --git a/xtask/src/task/manifest/remote.rs b/xtask/src/task/manifest/remote.rs new file mode 100644 index 00000000..a320df83 --- /dev/null +++ b/xtask/src/task/manifest/remote.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 + +use crate::task::manifest::{download_manifest::*, util}; +use anyhow::Result; +use regex::Regex; +use serde::Deserialize; +use std::{collections::HashMap, sync::LazyLock}; + +// Objects parsed from the JSON returned by GitHub's Release API + +#[derive(Clone, Debug, Deserialize)] +struct RawAssetDigest { + pub name: String, + pub size: usize, + pub browser_download_url: url::Url, +} + +#[derive(Clone, Debug, Deserialize)] +struct RawReleaseDigest { + pub name: String, + pub assets: Vec, +} + +#[derive(Clone, Debug)] +pub struct ReleaseDigest(pub HashMap>); + +// Used on the name of a RawReleaseDigest to extract release's name and version +static PLUGIN_RELEASE_REGEX: LazyLock = + LazyLock::new(|| Regex::new(r"(.*)-[v]?([0-9]+.[0-9]+.[0-9]+)").unwrap()); + +const SHA256_SUFFIX: &str = ".sha256"; + +impl TryFrom> for ReleaseDigest { + type Error = anyhow::Error; + + fn try_from(value: Vec) -> Result { + let mut releases = HashMap::>::new(); + + // For each RawReleaseDigest, + for release in value { + // Attempt to split out the name and version string from the raw name, + // this should fail for non-plugins because they follow a different release + // naming convention + let Some((name, version)) = + PLUGIN_RELEASE_REGEX.captures(&release.name).and_then(|m| { + parse_plugin_version(m.get(2).unwrap().as_str()) + .map(|v| (m.get(1).unwrap().as_str().to_owned(), v)) + }) + else { + continue; + }; + + let plugin_entry: &mut _ = match releases.get_mut(&name) { + Some(a) => a, + None => { + releases.insert(name.clone(), vec![]); + releases.get_mut(&name).unwrap() + } + }; + + // Create a lookup table of asset name to index in release.assets to + // easily find the entry that a given .sha256 file is related to + let asset_idxs: HashMap = HashMap::from_iter( + release + .assets + .iter() + .enumerate() + .map(|(i, a)| (a.name.clone(), i)), + ); + + let name_prefix = format!("{name}-"); + + // Look for files matching {name}-{arch}.{archive}.sha256 + for asset in release.assets.iter() { + let Some(no_name) = asset.name.strip_prefix(&name_prefix) else { + continue; + }; + let Some(no_hash_ext) = no_name.strip_suffix(SHA256_SUFFIX) else { + continue; + }; + + // The file related to this .sha256 file will be called `{name}-{arch}.{archive}` + let tgt_name = asset.name.strip_suffix(SHA256_SUFFIX).unwrap().to_owned(); + let Some(idx) = asset_idxs.get(&tgt_name) else { + eprintln!("warning: sha256 sum file found without corresponding asset"); + continue; + }; + + let tgt = release.assets.get(*idx).unwrap(); + + // Store the URL from which we can get the sha256 if we need it later + let digest = BufferedDigest::Remote(asset.browser_download_url.clone()); + + let Some((arch, fmt_str)) = no_hash_ext.split_once('.') else { + eprintln!("warning: malformed sha256 sum file name {}", asset.name); + continue; + }; + + let Ok(format): Result = fmt_str.try_into() else { + eprintln!("warning: malformed sha256 sum file name {}", asset.name); + continue; + }; + + plugin_entry.push(DownloadManifestEntry { + version: version.clone(), + arch: Arch(arch.to_owned()), + url: tgt.browser_download_url.clone(), + size: Size::new(tgt.size as u64), + hash: HashWithDigest { + hash_algorithm: HashAlgorithm::Sha256, + digest, + }, + compress: Compress { format }, + }); + } + } + + Ok(ReleaseDigest(releases)) + } +} + +pub fn get_hipcheck_releases(github_api_token: &str) -> Result { + let auth_agent = util::authenticated_agent::AuthenticatedAgent::new(github_api_token); + let raw_rel_json = auth_agent + .get("https://api.github.com/repos/mitre/hipcheck/releases") + .call()? + .into_string()?; + let raw_digest: Vec = serde_json::from_str(&raw_rel_json)?; + raw_digest.try_into() +} diff --git a/xtask/src/task/manifest/util/agent.rs b/xtask/src/task/manifest/util/agent.rs new file mode 100644 index 00000000..98d88cbe --- /dev/null +++ b/xtask/src/task/manifest/util/agent.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Globally defined agent containing system TLS Certs. + +use rustls::{ClientConfig, RootCertStore}; +use std::sync::{Arc, OnceLock}; +use ureq::{Agent, AgentBuilder}; + +/// Global static holding the agent with the appropriate TLS certs. +static AGENT: OnceLock = OnceLock::new(); + +/// Get or initialize the global static agent used in making http(s) requests for hipcheck. +/// +/// # Panics +/// - If native certs cannot be loaded the first time this function is called. +pub fn agent() -> &'static Agent { + AGENT.get_or_init(|| { + // Retrieve system certs + let mut roots = RootCertStore::empty(); + let native_certs = + rustls_native_certs::load_native_certs().expect("should load native certs"); + roots.add_parsable_certificates(native_certs); + + // Add certs to connection configuration + let tls_config = ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + + // Construct agent + AgentBuilder::new().tls_config(Arc::new(tls_config)).build() + }) +} diff --git a/xtask/src/task/manifest/util/authenticated_agent.rs b/xtask/src/task/manifest/util/authenticated_agent.rs new file mode 100644 index 00000000..d8bca119 --- /dev/null +++ b/xtask/src/task/manifest/util/authenticated_agent.rs @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Defines an authenticated [`Agent`] type that adds token auth to all requests. + +use crate::task::manifest::util::{agent, redacted::Redacted}; +use ureq::{Agent, Request}; + +/// An [`Agent`] which authenticates requests with token auth. +/// +/// This wrapper is used to work around the fact that `ureq` removed functionality +/// to do this as part of the [`Agent`] type directly. +pub struct AuthenticatedAgent<'token> { + /// The agent used to make the request. + agent: &'static Agent, + + /// The token to use with each request. + token: Redacted<&'token str>, +} + +impl<'token> AuthenticatedAgent<'token> { + /// Construct a new authenticated agent. + pub fn new(token: &'token str) -> AuthenticatedAgent<'token> { + AuthenticatedAgent { + agent: agent::agent(), + token: Redacted::new(token), + } + } + + /// Make an authenticated GET request. + pub fn get(&self, path: &str) -> Request { + self.agent.get(path).token_auth(self.token.as_ref()) + } + + /// Make an authenticated POST request. + #[allow(unused)] + pub fn post(&self, path: &str) -> Request { + self.agent.post(path).token_auth(self.token.as_ref()) + } +} + +/// The key to use for the authorization HTTP header. +const AUTH_KEY: &str = "Authorization"; + +/// Extension trait to add a convenient "token auth" method. +trait TokenAuth { + /// Sets a token authentication header on a request. + fn token_auth(self, token: &str) -> Self; +} + +impl TokenAuth for Request { + fn token_auth(self, token: &str) -> Self { + self.set(AUTH_KEY, &format!("token {}", token)) + } +} diff --git a/xtask/src/task/manifest/util/mod.rs b/xtask/src/task/manifest/util/mod.rs new file mode 100644 index 00000000..b9cf7777 --- /dev/null +++ b/xtask/src/task/manifest/util/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 + +pub mod agent; +pub mod authenticated_agent; +pub mod redacted; diff --git a/xtask/src/task/manifest/util/redacted.rs b/xtask/src/task/manifest/util/redacted.rs new file mode 100644 index 00000000..6e590205 --- /dev/null +++ b/xtask/src/task/manifest/util/redacted.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Utility type for hiding data from the user when printed in a debug message. + +use std::fmt::{Debug, Formatter, Result as FmtResult}; + +/// Helper container to ensure a value isn't printed. +pub struct Redacted(T); + +impl Redacted { + /// Construct a new redacted value. + pub fn new(val: T) -> Redacted { + Redacted(val) + } +} + +impl AsRef for Redacted { + fn as_ref(&self) -> &T { + &self.0 + } +} + +impl AsMut for Redacted { + fn as_mut(&mut self) -> &mut T { + &mut self.0 + } +} + +impl Debug for Redacted { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + write!(f, "") + } +} diff --git a/xtask/src/task/mod.rs b/xtask/src/task/mod.rs index a7d1a5c6..f944fc57 100644 --- a/xtask/src/task/mod.rs +++ b/xtask/src/task/mod.rs @@ -8,3 +8,4 @@ pub mod check; pub mod ci; pub mod rfd; pub mod site; +pub mod manifest;