Consolidate linux and windows containerd config templates

pkg/agent/config/config.go

@@ -22,7 +22,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/k3s-io/k3s/pkg/agent/containerd"
 	"github.com/k3s-io/k3s/pkg/agent/proxy"
 	agentutil "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
@@ -639,51 +638,23 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
 	nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
 	nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController
-	if envInfo.Rootless {
-		nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "agent", "kubelet")
-	}
 	nodeConfig.AgentConfig.Snapshotter = envInfo.Snapshotter
 	nodeConfig.AgentConfig.IPSECPSK = controlConfig.IPSECPSK
 	nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml")
 	nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "agent", "containerd")
 	nodeConfig.CRIDockerd.Root = filepath.Join(envInfo.DataDir, "agent", "cri-dockerd")
-	if !nodeConfig.Docker {
-		if nodeConfig.ImageServiceEndpoint != "" {
-			nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ImageServiceEndpoint
-		} else if nodeConfig.ContainerRuntimeEndpoint == "" {
-			switch nodeConfig.AgentConfig.Snapshotter {
-			case "overlayfs":
-				if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil {
-					return nil, errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"",
-						nodeConfig.Containerd.Root)
-				}
-			case "fuse-overlayfs":
-				if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil {
-					return nil, errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"",
-						nodeConfig.Containerd.Root)
-				}
-			case "stargz":
-				if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil {
-					return nil, errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
-						nodeConfig.Containerd.Root)
-				}
-				nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
-			}
-		} else {
-			nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ContainerRuntimeEndpoint
-		}
-	}
 	nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd")
 	nodeConfig.Containerd.Log = filepath.Join(envInfo.DataDir, "agent", "containerd", "containerd.log")
 	nodeConfig.Containerd.Registry = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "certs.d")
 	nodeConfig.Containerd.NoDefault = envInfo.ContainerdNoDefault
 	nodeConfig.Containerd.NonrootDevices = envInfo.ContainerdNonrootDevices
 	nodeConfig.Containerd.Debug = envInfo.Debug
-	applyContainerdStateAndAddress(nodeConfig)
-	applyCRIDockerdAddress(nodeConfig)
-	applyContainerdQoSClassConfigFileIfPresent(envInfo, &nodeConfig.Containerd)
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl")
 
+	if envInfo.Rootless {
+		nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "agent", "kubelet")
+	}
+
 	if envInfo.BindAddress != "" {
 		nodeConfig.AgentConfig.ListenAddress = envInfo.BindAddress
 	} else {
@@ -739,13 +710,26 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		}
 	}
 
-	if nodeConfig.Docker {
+	if nodeConfig.ImageServiceEndpoint != "" {
+		nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ImageServiceEndpoint
+	}
+
+	if nodeConfig.ContainerRuntimeEndpoint != "" {
+		nodeConfig.AgentConfig.RuntimeSocket = nodeConfig.ContainerRuntimeEndpoint
+	} else if nodeConfig.Docker {
+		if err := applyCRIDockerdOSSpecificConfig(nodeConfig); err != nil {
+			return nil, err
+		}
 		nodeConfig.AgentConfig.CNIPlugin = true
 		nodeConfig.AgentConfig.RuntimeSocket = nodeConfig.CRIDockerd.Address
-	} else if nodeConfig.ContainerRuntimeEndpoint == "" {
-		nodeConfig.AgentConfig.RuntimeSocket = nodeConfig.Containerd.Address
 	} else {
-		nodeConfig.AgentConfig.RuntimeSocket = nodeConfig.ContainerRuntimeEndpoint
+		if err := applyContainerdOSSpecificConfig(nodeConfig); err != nil {
+			return nil, err
+		}
+		if err := applyContainerdQoSClassConfigFileIfPresent(envInfo, &nodeConfig.Containerd); err != nil {
+			return nil, err
+		}
+		nodeConfig.AgentConfig.RuntimeSocket = nodeConfig.Containerd.Address
 	}
 
 	if controlConfig.ClusterIPRange != nil {

pkg/agent/config/config_linux.go

@@ -4,25 +4,47 @@
 package config
 
 import (
-	"errors"
 	"os"
 	"path/filepath"
 
+	"github.com/k3s-io/k3s/pkg/agent/containerd"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
-func applyContainerdStateAndAddress(nodeConfig *config.Node) {
+// applyContainerdOSSpecificConfig sets linux-specific containerd config
+func applyContainerdOSSpecificConfig(nodeConfig *config.Node) error {
 	nodeConfig.Containerd.State = "/run/k3s/containerd"
 	nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
-}
 
-func applyCRIDockerdAddress(nodeConfig *config.Node) {
-	nodeConfig.CRIDockerd.Address = "unix:///run/k3s/cri-dockerd/cri-dockerd.sock"
+	// validate that the selected snapshotter supports the filesystem at the root path.
+	// for stargz, also overrides the image service endpoint path.
+	switch nodeConfig.AgentConfig.Snapshotter {
+	case "overlayfs":
+		if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil {
+			return errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"",
+				nodeConfig.Containerd.Root)
+		}
+	case "fuse-overlayfs":
+		if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil {
+			return errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"",
+				nodeConfig.Containerd.Root)
+		}
+	case "stargz":
+		if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil {
+			return errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
+				nodeConfig.Containerd.Root)
+		}
+		nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+	}
+
+	return nil
 }
 
-func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdConfig *config.Containerd) {
+// applyContainerdQoSClassConfigFileIfPresent sets linux-specific qos config
+func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdConfig *config.Containerd) error {
 	containerdConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "containerd")
 
 	blockioPath := filepath.Join(containerdConfigDir, "blockio_config.yaml")
@@ -44,6 +66,14 @@ func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdC
 			containerdConfig.RDTConfig = rdtPath
 		}
 	}
+
+	return nil
+}
+
+// applyCRIDockerdOSSpecificConfig sets linux-specific cri-dockerd config
+func applyCRIDockerdOSSpecificConfig(nodeConfig *config.Node) error {
+	nodeConfig.CRIDockerd.Address = "unix:///run/k3s/cri-dockerd/cri-dockerd.sock"
+	return nil
 }
 
 // configureACL will configure an Access Control List for the specified file.

pkg/agent/config/config_windows.go

@@ -15,17 +15,25 @@ import (
 	"golang.org/x/sys/windows"
 )
 
-func applyContainerdStateAndAddress(nodeConfig *config.Node) {
+// applyContainerdOSSpecificConfig sets windows-specific containerd config
+func applyContainerdOSSpecificConfig(nodeConfig *config.Node) error {
+	nodeConfig.AgentConfig.Snapshotter = "windows"
 	nodeConfig.Containerd.State = filepath.Join(nodeConfig.Containerd.Root, "state")
 	nodeConfig.Containerd.Address = "npipe:////./pipe/containerd-containerd"
+	nodeConfig.DefaultRuntime = "runhcs-wcow-process"
+	return nil
 }
 
-func applyCRIDockerdAddress(nodeConfig *config.Node) {
-	nodeConfig.CRIDockerd.Address = "npipe:////.pipe/cri-dockerd"
+// applyContainerdQoSClassConfigFileIfPresent sets windows-specific qos config
+func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdConfig *config.Containerd) error {
+	// QoS-class resource management not supported on windows.
+	return nil
 }
 
-func applyContainerdQoSClassConfigFileIfPresent(envInfo *cmds.Agent, containerdConfig *config.Containerd) {
-	// QoS-class resource management not supported on windows.
+// applyCRIDockerdOSSpecificConfig sets windows-specific cri-dockerd config
+func applyCRIDockerdOSSpecificConfig(nodeConfig *config.Node) error {
+	nodeConfig.CRIDockerd.Address = "npipe:////.pipe/cri-dockerd"
+	return nil
 }
 
 // configureACL will configure an Access Control List for the specified file,

pkg/agent/containerd/config_windows.go

@@ -31,8 +31,6 @@ func SetupContainerdConfig(cfg *config.Node) error {
 	containerdConfig := templates.ContainerdConfig{
 		NodeConfig:            cfg,
 		DisableCgroup:         true,
-		SystemdCgroup:         false,
-		IsRunningInUserNS:     false,
 		PrivateRegistryConfig: cfg.AgentConfig.Registry,
 		NoDefaultEndpoint:     cfg.Containerd.NoDefault,
 	}

pkg/agent/templates/templates.go

@@ -42,6 +42,108 @@ type HostConfig struct {
 	Endpoints []RegistryEndpoint
 }
 
+const ContainerdConfigTemplate = `
+{{- /* */ -}}
+# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
+version = 2
+root = {{ printf "%q" .NodeConfig.Containerd.Root }}
+state = {{ printf "%q" .NodeConfig.Containerd.State }}
+
+[plugins."io.containerd.internal.v1.opt"]
+  path = {{ printf "%q" .NodeConfig.Containerd.Opt }}
+
+[plugins."io.containerd.grpc.v1.cri"]
+  stream_server_address = "127.0.0.1"
+  stream_server_port = "10010"
+  enable_selinux = {{ .NodeConfig.SELinux }}
+  enable_unprivileged_ports = {{ .EnableUnprivileged }}
+  enable_unprivileged_icmp = {{ .EnableUnprivileged }}
+  device_ownership_from_security_context = {{ .NonrootDevices }}
+
+{{- if .DisableCgroup}}
+  disable_cgroup = true
+{{end}}
+{{- if .IsRunningInUserNS }}
+  disable_apparmor = true
+  restrict_oom_score_adj = true
+{{end}}
+
+{{- if .NodeConfig.AgentConfig.PauseImage }}
+  sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
+{{end}}
+
+{{- if .NodeConfig.AgentConfig.Snapshotter }}
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}"
+  disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}}
+  {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}}
+{{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}
+{{ if .NodeConfig.AgentConfig.ImageServiceSocket }}
+[plugins."io.containerd.snapshotter.v1.stargz"]
+cri_keychain_image_service_path = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}"
+[plugins."io.containerd.snapshotter.v1.stargz".cri_keychain]
+enable_keychain = true
+{{end}}
+
+[plugins."io.containerd.snapshotter.v1.stargz".registry]
+  config_path = "{{ .NodeConfig.Containerd.Registry }}"
+
+{{ if .PrivateRegistryConfig }}
+{{range $k, $v := .PrivateRegistryConfig.Configs }}
+{{ if $v.Auth }}
+[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".auth]
+  {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
+  {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
+  {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
+  {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
+{{end}}
+{{end}}
+{{end}}
+{{end}}
+{{end}}
+
+{{- if not .NodeConfig.NoFlannel }}
+[plugins."io.containerd.grpc.v1.cri".cni]
+  bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
+  conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
+{{end}}
+
+{{- if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }}
+[plugins."io.containerd.service.v1.tasks-service"]
+  {{ if .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = "{{ .NodeConfig.Containerd.BlockIOConfig }}"{{end}}
+  {{ if .NodeConfig.Containerd.RDTConfig }}rdt_config_file = "{{ .NodeConfig.Containerd.RDTConfig }}"{{end}}
+{{end}}
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+  SystemdCgroup = {{ .SystemdCgroup }}
+
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "{{ .NodeConfig.Containerd.Registry }}"
+
+{{ if .PrivateRegistryConfig }}
+{{range $k, $v := .PrivateRegistryConfig.Configs }}
+{{ if $v.Auth }}
+[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth]
+  {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
+  {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}}
+  {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
+  {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
+{{end}}
+{{end}}
+{{end}}
+
+{{range $k, $v := .ExtraRuntimes}}
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}"]
+  runtime_type = "{{$v.RuntimeType}}"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}".options]
+  BinaryName = "{{$v.BinaryName}}"
+  SystemdCgroup = {{ $.SystemdCgroup }}
+{{end}}
+`
+
 var HostsTomlHeader = "# File generated by " + version.Program + ". DO NOT EDIT.\n"
 
 const HostsTomlTemplate = `