diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index f0a0f4471ee7..57e21c6731c8 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -25,6 +25,7 @@ slog-scope = "4.4.0" serde_json = "1.0.73" thiserror = "1.0" toml = "0.5.8" +serde-enum-str = "0.4" oci = { path = "../oci" } safe-path = { path = "../safe-path" } diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 89da372de845..0b17b10628ef 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -15,7 +15,7 @@ use serde::Deserialize; use crate::config::default::DEFAULT_AGENT_TYPE_NAME; use crate::config::default::DEFAULT_HYPERVISOR; use crate::config::default::DEFAULT_RUNTIME_NAME; -use crate::config::hypervisor::get_hypervisor_plugin; +use crate::config::hypervisor::{get_hypervisor_plugin, HugePageType}; use crate::config::TomlConfig; use crate::sl; @@ -225,8 +225,11 @@ pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS: &str = pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC: &str = "io.katacontainers.config.hypervisor.enable_mem_prealloc"; /// A sandbox annotation to specify if the memory should be pre-allocated from huge pages. -pub const KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES: &str = "io.katacontainers.config.hypervisor.enable_hugepages"; +/// A sandbox annotation to specify huge page mode of memory backend. +pub const KATA_ANNO_CFG_HYPERVISOR_HUGEPAGE_TYPE: &str = + "io.katacontainers.config.hypervisor.hugepage_type"; /// A sandbox annotation to soecify file based memory backend root directory. pub const KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR: &str = "io.katacontainers.config.hypervisor.file_mem_backend"; @@ -740,14 +743,29 @@ impl Annotation { return Err(bool_err); } }, - KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES => match self.get_value::(key) { - Ok(r) => { - hv.memory_info.enable_hugepages = r.unwrap_or_default(); + KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES => { + match self.get_value::(key) { + Ok(r) => { + hv.memory_info.enable_hugepages = r.unwrap_or_default(); + } + Err(_e) => { + return Err(bool_err); + } } - Err(_e) => { - return Err(bool_err); + } + KATA_ANNO_CFG_HYPERVISOR_HUGEPAGE_TYPE => { + match self.get_value::(key) { + Ok(r) => { + hv.memory_info.hugepage_type = r.unwrap_or_default(); + } + Err(e) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("parse huge pages type: {}, error: {}", value, e), + )); + } } - }, + } KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => { hv.memory_info.validate_memory_backend_path(value)?; hv.memory_info.file_mem_backend = value.to_string(); diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 9056a0dd00cd..19fb83d61a25 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -29,6 +29,7 @@ use std::sync::{Arc, Mutex}; use lazy_static::lazy_static; use regex::RegexSet; +use serde_enum_str::{Deserialize_enum_str, Serialize_enum_str}; use super::{default, ConfigOps, ConfigPlugin, TomlConfig}; use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX; @@ -540,6 +541,25 @@ impl MachineInfo { } } +/// Huge page type for VM RAM backend +#[derive(Clone, Debug, Deserialize_enum_str, Serialize_enum_str, PartialEq, Eq)] +pub enum HugePageType { + /// This will result in the VM memory being allocated using hugetlbfs backend. This is useful + /// when you want to use vhost-user network stacks within the container. This will automatically + /// result in memory pre allocation. + #[serde(rename = "hugetlbfs")] + Hugetlbfs, + /// This will result in the VM memory being allocated using transparant huge page backend. + #[serde(rename = "thp")] + THP, +} + +impl Default for HugePageType { + fn default() -> Self { + Self::Hugetlbfs + } +} + /// Virtual machine memory configuration information. #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct MemoryInfo { @@ -577,12 +597,18 @@ pub struct MemoryInfo { /// Enable huge pages for VM RAM, default false /// - /// Enabling this will result in the VM memory being allocated using huge pages. This is useful - /// when you want to use vhost-user network stacks within the container. This will automatically - /// result in memory pre allocation. + /// Enabling this will result in the VM memory being allocated using huge pages. + /// Its backend type is specified by item "hugepage_type" #[serde(default)] pub enable_hugepages: bool, + /// Select huge page type, default "hugetlbfs" + /// Following huge types are supported: + /// - hugetlbfs + /// - thp + #[serde(default)] + pub hugepage_type: HugePageType, + /// Specifies virtio-mem will be enabled or not. /// /// Please note that this option should be used with the command diff --git a/src/libs/kata-types/tests/test_config.rs b/src/libs/kata-types/tests/test_config.rs index 800a05f70bd4..d350df724bb9 100644 --- a/src/libs/kata-types/tests/test_config.rs +++ b/src/libs/kata-types/tests/test_config.rs @@ -10,15 +10,15 @@ mod tests { KATA_ANNO_CFG_EXPERIMENTAL, KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH, KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER, KATA_ANNO_CFG_HYPERVISOR_CTLPATH, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS, - KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS, - KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP, KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR, - KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES, - KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH, KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH, - KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC, KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS, - KATA_ANNO_CFG_HYPERVISOR_PATH, KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, - KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, - KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM, KATA_ANNO_CFG_KERNEL_MODULES, - KATA_ANNO_CFG_RUNTIME_NAME, + KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES, + KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS, KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP, + KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR, + KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH, + KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH, KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC, + KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS, KATA_ANNO_CFG_HYPERVISOR_PATH, + KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON, + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM, + KATA_ANNO_CFG_KERNEL_MODULES, KATA_ANNO_CFG_RUNTIME_NAME, }; use kata_types::config::KataConfig; use kata_types::config::{QemuConfig, TomlConfig}; @@ -138,7 +138,7 @@ mod tests { "./test_file_backend_mem_root".to_string(), ); anno_hash.insert( - KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES.to_string(), + KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES.to_string(), "false".to_string(), ); anno_hash.insert( diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 6823dac9bfbb..68d4d7fbe209 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -7,7 +7,7 @@ use super::vmm_instance::VmmInstance; use crate::{ device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState, - DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, + DEV_HUGEPAGES, HUGETLBFS, HUGE_SHMEM, HYPERVISOR_DRAGONBALL, SHMEM, }; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; @@ -19,7 +19,10 @@ use dragonball::{ use kata_sys_util::mount; use kata_types::{ capabilities::{Capabilities, CapabilityBits}, - config::{hypervisor::Hypervisor as HypervisorConfig, KATA_PATH}, + config::{ + hypervisor::{HugePageType, Hypervisor as HypervisorConfig}, + KATA_PATH, + }, }; use nix::mount::MsFlags; use persist::sandbox_persist::Persist; @@ -175,7 +178,10 @@ impl DragonballInner { fn set_vm_base_config(&mut self) -> Result<()> { let serial_path = [&self.run_dir, "console.sock"].join("/"); let (mem_type, mem_file_path) = if self.config.memory_info.enable_hugepages { - (String::from(HUGETLBFS), String::from(DEV_HUGEPAGES)) + match self.config.memory_info.hugepage_type { + HugePageType::THP => (String::from(HUGE_SHMEM), String::from("")), + HugePageType::Hugetlbfs => (String::from(HUGETLBFS), String::from(DEV_HUGEPAGES)), + } } else { (String::from(SHMEM), String::from("")) }; diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 69f9fe9b4155..9e6184b74b2a 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -52,6 +52,7 @@ const VM_ROOTFS_FILESYSTEM_EROFS: &str = "erofs"; const DEV_HUGEPAGES: &str = "/dev/hugepages"; pub const HUGETLBFS: &str = "hugetlbfs"; const SHMEM: &str = "shmem"; +const HUGE_SHMEM: &str = "hugeshmem"; pub const HYPERVISOR_DRAGONBALL: &str = "dragonball"; pub const HYPERVISOR_QEMU: &str = "qemu";