From 942962bf005a7036e04f4e572f3434f476cb567c Mon Sep 17 00:00:00 2001 From: Noel Georgi Date: Thu, 7 Nov 2024 19:23:16 +0530 Subject: [PATCH] docs: add docs on usernamespace support in k8s Add docs and test for usernamespaces support in Kubernetes. Fixes: #8554 Signed-off-by: Noel Georgi --- .github/workflows/ci.yaml | 9 +- .../workflows/integration-misc-4-cron.yaml | 9 +- .kres.yaml | 7 + hack/release.toml | 7 + hack/test/patches/usernamespace.yaml | 13 ++ .../k8s/testdata/usernamespace.yaml | 12 ++ internal/integration/k8s/usernamespace.go | 128 ++++++++++++++++++ .../configuration/usernamespace.md | 30 ++++ 8 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 hack/test/patches/usernamespace.yaml create mode 100644 internal/integration/k8s/testdata/usernamespace.yaml create mode 100644 internal/integration/k8s/usernamespace.go create mode 100644 website/content/v1.9/kubernetes-guides/configuration/usernamespace.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e85b53a809..8f163b2e8a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-10-23T17:20:56Z by kres 6d3cad4. +# Generated on 2024-11-07T15:01:48Z by kres 1fc767a. name: default concurrency: @@ -2407,6 +2407,13 @@ jobs: WITH_APPARMOR_LSM_ENABLED: "yes" run: | sudo -E make e2e-qemu + - name: e2e-k8s-user-namespace + env: + IMAGE_REGISTRY: registry.dev.siderolabs.io + SHORT_INTEGRATION_TEST: "yes" + WITH_CONFIG_PATCH: '@hack/test/patches/usernamespace.yaml' + run: | + sudo -E make e2e-qemu - name: save artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/integration-misc-4-cron.yaml b/.github/workflows/integration-misc-4-cron.yaml index 2d2b2a5ad9..c45e44b945 100644 --- a/.github/workflows/integration-misc-4-cron.yaml +++ b/.github/workflows/integration-misc-4-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-09-12T16:43:46Z by kres 8be5fa7. +# Generated on 2024-11-07T15:01:48Z by kres 1fc767a. name: integration-misc-4-cron concurrency: @@ -109,6 +109,13 @@ jobs: WITH_APPARMOR_LSM_ENABLED: "yes" run: | sudo -E make e2e-qemu + - name: e2e-k8s-user-namespace + env: + IMAGE_REGISTRY: registry.dev.siderolabs.io + SHORT_INTEGRATION_TEST: "yes" + WITH_CONFIG_PATCH: '@hack/test/patches/usernamespace.yaml' + run: | + sudo -E make e2e-qemu - name: save artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.kres.yaml b/.kres.yaml index a959ebc37f..de117abca3 100644 --- a/.kres.yaml +++ b/.kres.yaml @@ -953,6 +953,13 @@ spec: SHORT_INTEGRATION_TEST: yes WITH_APPARMOR_LSM_ENABLED: yes IMAGE_REGISTRY: registry.dev.siderolabs.io + - name: e2e-k8s-user-namespace + command: e2e-qemu + withSudo: true + environment: + SHORT_INTEGRATION_TEST: yes + WITH_CONFIG_PATCH: "@hack/test/patches/usernamespace.yaml" + IMAGE_REGISTRY: registry.dev.siderolabs.io - name: save-talos-logs conditions: - always diff --git a/hack/release.toml b/hack/release.toml index affaa81ace..3340d47d6c 100644 --- a/hack/release.toml +++ b/hack/release.toml @@ -25,6 +25,13 @@ Kubernetes: 1.32.0-beta.0 runc: 1.2.1 Talos is built with Go 1.23.2. +""" + + [notes.usernamespaces] + title = "User Namespaces" + description = """\ +Talos Linux now supports running Kubernetes pods with user namespaces enabled. +Refer to the [documentation](https://www.talos.dev/v1.9/kubernetes-guides/configuration/usernamespace/) for more information. """ [notes.apparmor] diff --git a/hack/test/patches/usernamespace.yaml b/hack/test/patches/usernamespace.yaml new file mode 100644 index 0000000000..5200d7fa1f --- /dev/null +++ b/hack/test/patches/usernamespace.yaml @@ -0,0 +1,13 @@ +--- +cluster: + apiServer: + extraArgs: + feature-gates: UserNamespacesSupport=true,UserNamespacesPodSecurityStandards=true +machine: + sysctls: + user.max_user_namespaces: "11255" + kubelet: + extraConfig: + featureGates: + UserNamespacesSupport: true + UserNamespacesPodSecurityStandards: true diff --git a/internal/integration/k8s/testdata/usernamespace.yaml b/internal/integration/k8s/testdata/usernamespace.yaml new file mode 100644 index 0000000000..3feaa32104 --- /dev/null +++ b/internal/integration/k8s/testdata/usernamespace.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: userns + namespace: default +spec: + hostUsers: false + containers: + - name: userns + command: ["/bin/sh", "-c", "--"] + args: ["trap : TERM INT; (sleep 1000) & wait"] + image: alpine diff --git a/internal/integration/k8s/usernamespace.go b/internal/integration/k8s/usernamespace.go new file mode 100644 index 0000000000..8bb2296a1e --- /dev/null +++ b/internal/integration/k8s/usernamespace.go @@ -0,0 +1,128 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build integration_k8s + +package k8s + +import ( + "bufio" + "bytes" + "context" + _ "embed" + "fmt" + "strings" + "time" + + "github.com/siderolabs/talos/internal/integration/base" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/config/machine" +) + +// UserNamespaceSuite verifies that a pod with user namespace works. +type UserNamespaceSuite struct { + base.K8sSuite +} + +//go:embed testdata/usernamespace.yaml +var userNamespacePodSpec []byte + +// SuiteName returns the name of the suite. +func (suite *UserNamespaceSuite) SuiteName() string { + return "k8s.UserNamespaceSuite" +} + +// TestUserNamespace verifies that a pod with user namespace works. +// +//nolint:gocyclo +func (suite *UserNamespaceSuite) TestUserNamespace() { + if suite.Cluster == nil { + suite.T().Skip("without full cluster state reaching out to the node IP is not reliable") + } + + if suite.Cluster.Provisioner() != base.ProvisionerQEMU { + suite.T().Skip("skipping usernamespace test since provisioner is not qemu") + } + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + suite.T().Cleanup(cancel) + + node := suite.RandomDiscoveredNodeInternalIP(machine.TypeWorker) + + nodeCtx := client.WithNodes(ctx, node) + + reader, err := suite.Client.Read(nodeCtx, "/proc/sys/user/max_user_namespaces") + suite.Require().NoError(err) + + var maxUserNamespaces bytes.Buffer + + _, err = maxUserNamespaces.ReadFrom(reader) + suite.Require().NoError(err) + + if strings.TrimSpace(maxUserNamespaces.String()) == "0" { + suite.T().Skip("skipping test since user namespace is disabled") + } + + usernamespacePodManifest := suite.ParseManifests(userNamespacePodSpec) + + suite.T().Cleanup(func() { + cleanUpCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute) + defer cleanupCancel() + + suite.DeleteManifests(cleanUpCtx, usernamespacePodManifest) + }) + + suite.ApplyManifests(ctx, usernamespacePodManifest) + + suite.Require().NoError(suite.WaitForPodToBeRunning(ctx, time.Minute, "default", "userns")) + + processResp, err := suite.Client.Processes(nodeCtx) + suite.Require().NoError(err) + + var sleepProcessPID int + + for _, processInfo := range processResp.Messages { + for _, process := range processInfo.Processes { + if strings.Contains(process.Args, "sleep 1000") { + sleepProcessPID = int(process.Pid) + + break + } + } + } + + suite.Require().NotZero(sleepProcessPID, "sleep process not found for user namespace test") + + reader, err = suite.Client.Read(nodeCtx, fmt.Sprintf("/proc/%d/status", sleepProcessPID)) + suite.Require().NoError(err) + + var processStatus bytes.Buffer + + _, err = processStatus.ReadFrom(reader) + suite.Require().NoError(err) + + scanner := bufio.NewScanner(&processStatus) + + var processUsingUserNamespace bool + + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "Uid:") { + fields := strings.Fields(line) + + if fields[0] != "0" && fields[1] != "0" && fields[2] != "0" && fields[3] != "0" { + processUsingUserNamespace = true + } + + break + } + } + + suite.Require().True(processUsingUserNamespace, "sleep process should not have root UID in host namespace\n", processStatus.String()) +} + +func init() { + allSuites = append(allSuites, new(UserNamespaceSuite)) +} diff --git a/website/content/v1.9/kubernetes-guides/configuration/usernamespace.md b/website/content/v1.9/kubernetes-guides/configuration/usernamespace.md new file mode 100644 index 0000000000..f8503efc1a --- /dev/null +++ b/website/content/v1.9/kubernetes-guides/configuration/usernamespace.md @@ -0,0 +1,30 @@ +--- +title: "User Namespaces" +description: "Guide on how to configure Talos Cluster to support User Namespaces" +--- + +User Namespaces are a feature of the Linux kernel that allows unprivileged users to have their own range of UIDs and GIDs, without needing to be root. + +Refer to the [official documentation](https://kubernetes.io/docs/concepts/workloads/pods/user-namespaces/) for more information on Usernamespaces. + +## Enabling Usernamespaces + +To enable User Namespaces in Talos, you need to add the following configuration to Talos machine configuration: + +```yaml +--- +cluster: + apiServer: + extraArgs: + feature-gates: UserNamespacesSupport=true,UserNamespacesPodSecurityStandards=true +machine: + sysctls: + user.max_user_namespaces: "11255" + kubelet: + extraConfig: + featureGates: + UserNamespacesSupport: true + UserNamespacesPodSecurityStandards: true +``` + +After applying the configuration, refer to the [official documentation](https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/) to configure workloads to use User Namespaces.