From 4e0291fd0083bf922b693b00b7b6b7e90e701066 Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Tue, 5 Dec 2023 19:59:30 +0800 Subject: [PATCH] Shorten the default node lease --- .../v1alpha1/kwokctl_configuration_types.go | 2 +- .../config/v1alpha1/zz_generated.defaults.go | 2 +- pkg/kwok/controllers/node_lease_controller.go | 27 ++++++ pkg/kwokctl/cmd/create/cluster/cluster.go | 1 + .../runtime/binary/cluster_snapshot.go | 39 ++++++-- .../runtime/compose/cluster_snapshot.go | 79 +++++++++++++--- pkg/kwokctl/runtime/kind/cluster.go | 15 ++- pkg/kwokctl/runtime/kind/cluster_snapshot.go | 33 +++++-- .../docs/generated/kwokctl_create_cluster.md | 1 + test/e2e/helper/node.go | 3 + test/e2e/kwokctl/binary/kubectl_test.go | 12 +++ test/e2e/kwokctl/docker/kubectl_test.go | 12 +++ test/e2e/kwokctl/kind-podman/kubectl_test.go | 12 +++ test/e2e/kwokctl/kind/kubectl_test.go | 12 +++ test/e2e/kwokctl/nerdctl/kubectl_test.go | 12 +++ test/e2e/kwokctl/podman/kubectl_test.go | 12 +++ test/e2e/restart.go | 73 +++++++++++++++ test/e2e/snapshot.go | 92 +++++++++++++++++++ .../testdata/binary/create_cluster.txt | 2 +- .../binary/create_cluster_with_extra.txt | 2 +- .../binary/create_cluster_with_verbosity.txt | 2 +- .../testdata/binary/snapshot_restore_etcd.txt | 8 ++ .../kwokctl/testdata/binary/start_cluster.txt | 2 +- .../testdata/docker/create_cluster.txt | 2 +- .../docker/create_cluster_with_extra.txt | 2 +- .../docker/create_cluster_with_verbosity.txt | 2 +- .../testdata/docker/snapshot_restore_etcd.txt | 8 ++ .../kind-podman/snapshot_restore_etcd.txt | 2 + .../testdata/kind/snapshot_restore_etcd.txt | 2 + .../testdata/nerdctl/create_cluster.txt | 2 +- .../nerdctl/create_cluster_with_extra.txt | 2 +- .../nerdctl/create_cluster_with_verbosity.txt | 2 +- .../nerdctl/snapshot_restore_etcd.txt | 7 ++ .../testdata/podman/create_cluster.txt | 2 +- .../podman/create_cluster_with_extra.txt | 2 +- .../podman/create_cluster_with_verbosity.txt | 2 +- .../testdata/podman/snapshot_restore_etcd.txt | 8 ++ 37 files changed, 452 insertions(+), 48 deletions(-) create mode 100644 test/e2e/restart.go create mode 100644 test/e2e/snapshot.go diff --git a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go index 41789568f..83e851c77 100644 --- a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go +++ b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go @@ -398,7 +398,7 @@ type KwokctlConfigurationOptions struct { NodeStatusUpdateFrequencyMilliseconds int64 `json:"nodeStatusUpdateFrequencyMilliseconds,omitempty"` // NodeLeaseDurationSeconds is the duration the Kubelet will set on its corresponding Lease. - // +default=1200 + // +default=40 NodeLeaseDurationSeconds uint `json:"nodeLeaseDurationSeconds,omitempty"` // BindAddress is the address to bind to. diff --git a/pkg/apis/config/v1alpha1/zz_generated.defaults.go b/pkg/apis/config/v1alpha1/zz_generated.defaults.go index 0799c266a..f6a036c4a 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.defaults.go +++ b/pkg/apis/config/v1alpha1/zz_generated.defaults.go @@ -92,7 +92,7 @@ func SetObjectDefaults_KwokctlConfiguration(in *KwokctlConfiguration) { in.Options.NodeStatusUpdateFrequencyMilliseconds = 1200000 } if in.Options.NodeLeaseDurationSeconds == 0 { - in.Options.NodeLeaseDurationSeconds = 1200 + in.Options.NodeLeaseDurationSeconds = 40 } if in.Options.BindAddress == "" { in.Options.BindAddress = "0.0.0.0" diff --git a/pkg/kwok/controllers/node_lease_controller.go b/pkg/kwok/controllers/node_lease_controller.go index a7a3619cc..9e63e1842 100644 --- a/pkg/kwok/controllers/node_lease_controller.go +++ b/pkg/kwok/controllers/node_lease_controller.go @@ -217,6 +217,24 @@ func (c *NodeLeaseController) sync(ctx context.Context, nodeName string) { logger.Info("Creating lease") latestLease, err := c.ensureLease(ctx, nodeName) if err != nil { + if apierrors.IsAlreadyExists(err) { + logger.Error("failed to create lease, lease already exists", err) + + _, err = c.syncLease(ctx, nodeName) + if err != nil { + logger.Error("failed to sync lease", err) + return + } + if c.onNodeManagedFunc != nil { + if c.Held(nodeName) { + c.onNodeManagedFunc(nodeName) + } else { + logger.Warn("Lease not held") + } + } + return + } + if !apierrors.IsNotFound(err) || !c.latestLease.IsEmpty() { logger.Error("failed to create lease", err) return @@ -243,6 +261,15 @@ func (c *NodeLeaseController) sync(ctx context.Context, nodeName string) { } } +func (c *NodeLeaseController) syncLease(ctx context.Context, leaseName string) (*coordinationv1.Lease, error) { + lease, err := c.typedClient.CoordinationV1().Leases(corev1.NamespaceNodeLease).Get(ctx, leaseName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + c.latestLease.Store(leaseName, lease) + return lease, nil +} + // ensureLease creates a lease if it does not exist func (c *NodeLeaseController) ensureLease(ctx context.Context, leaseName string) (*coordinationv1.Lease, error) { lease := &coordinationv1.Lease{ diff --git a/pkg/kwokctl/cmd/create/cluster/cluster.go b/pkg/kwokctl/cmd/create/cluster/cluster.go index b5e462e08..ef380edb8 100644 --- a/pkg/kwokctl/cmd/create/cluster/cluster.go +++ b/pkg/kwokctl/cmd/create/cluster/cluster.go @@ -137,6 +137,7 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().StringVar(&flags.Kubeconfig, "kubeconfig", flags.Kubeconfig, "The path to the kubeconfig file will be added to the newly created cluster and set to current-context") cmd.Flags().BoolVar(&flags.Options.DisableQPSLimits, "disable-qps-limits", flags.Options.DisableQPSLimits, "Disable QPS limits for components") cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crds", flags.Options.EnableCRDs, "List of CRDs to enable") + cmd.Flags().UintVar(&flags.Options.NodeLeaseDurationSeconds, "node-lease-duration-seconds", flags.Options.NodeLeaseDurationSeconds, "Duration of node lease in seconds") return cmd } diff --git a/pkg/kwokctl/runtime/binary/cluster_snapshot.go b/pkg/kwokctl/runtime/binary/cluster_snapshot.go index 6b6540db2..42864d171 100644 --- a/pkg/kwokctl/runtime/binary/cluster_snapshot.go +++ b/pkg/kwokctl/runtime/binary/cluster_snapshot.go @@ -39,19 +39,44 @@ func (c *Cluster) SnapshotSave(ctx context.Context, path string) error { func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error { logger := log.FromContext(ctx) - err := c.StopComponent(ctx, consts.ComponentEtcd) - if err != nil { - logger.Error("Failed to stop etcd", err) + // Restart etcd and kube-apiserver + components := []string{ + consts.ComponentEtcd, + consts.ComponentKubeApiserver, } - defer func() { - err = c.StartComponent(ctx, consts.ComponentEtcd) + for _, component := range components { + err := c.StopComponent(ctx, component) if err != nil { - logger.Error("Failed to start etcd", err) + logger.Error("Failed to stop", err, "component", component) + } + } + defer func() { + for _, component := range components { + err := c.StartComponent(ctx, component) + if err != nil { + logger.Error("Failed to start", err, "component", component) + } + } + + components := []string{ + consts.ComponentKwokController, + consts.ComponentKubeControllerManager, + consts.ComponentKubeScheduler, + } + for _, component := range components { + err := c.StopComponent(ctx, component) + if err != nil { + logger.Error("Failed to stop", err, "component", component) + } + err = c.StartComponent(ctx, component) + if err != nil { + logger.Error("Failed to start", err, "component", component) + } } }() etcdDataTmp := c.GetWorkdirPath("etcd-data") - err = c.RemoveAll(etcdDataTmp) + err := c.RemoveAll(etcdDataTmp) if err != nil { return err } diff --git a/pkg/kwokctl/runtime/compose/cluster_snapshot.go b/pkg/kwokctl/runtime/compose/cluster_snapshot.go index 308fbc7ff..eb4e42973 100644 --- a/pkg/kwokctl/runtime/compose/cluster_snapshot.go +++ b/pkg/kwokctl/runtime/compose/cluster_snapshot.go @@ -71,15 +71,39 @@ func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error { etcdContainerName := c.Name() + "-etcd" if conf.Runtime != consts.RuntimeTypeNerdctl { - // Restart etcd container - err = c.StopComponent(ctx, consts.ComponentEtcd) - if err != nil { - logger.Error("Failed to stop etcd", err) + // Restart etcd and kube-apiserver + components := []string{ + consts.ComponentEtcd, + consts.ComponentKubeApiserver, } - defer func() { - err = c.StartComponent(ctx, consts.ComponentEtcd) + for _, component := range components { + err := c.StopComponent(ctx, component) if err != nil { - logger.Error("Failed to start etcd", err) + logger.Error("Failed to stop", err, "component", component) + } + } + defer func() { + for _, component := range components { + err := c.StartComponent(ctx, component) + if err != nil { + logger.Error("Failed to start", err, "component", component) + } + } + + components := []string{ + consts.ComponentKwokController, + consts.ComponentKubeControllerManager, + consts.ComponentKubeScheduler, + } + for _, component := range components { + err := c.StopComponent(ctx, component) + if err != nil { + logger.Error("Failed to stop", err, "component", component) + } + err = c.StartComponent(ctx, component) + if err != nil { + logger.Error("Failed to start", err, "component", component) + } } }() @@ -110,15 +134,42 @@ func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error { return err } - // Restart etcd container - err = c.StopComponent(ctx, consts.ComponentEtcd) - if err != nil { - logger.Error("Failed to stop etcd", err) + // Restart etcd and kube-apiserver + components := []string{ + consts.ComponentEtcd, } - defer func() { - err = c.StartComponent(ctx, consts.ComponentEtcd) + for _, component := range components { + err := c.StopComponent(ctx, component) if err != nil { - logger.Error("Failed to start etcd", err) + logger.Error("Failed to stop", err, "component", component) + } + } + defer func() { + components := []string{ + consts.ComponentEtcd, + consts.ComponentKubeApiserver, + } + for _, component := range components { + err := c.StartComponent(ctx, component) + if err != nil { + logger.Error("Failed to start", err, "component", component) + } + } + + components = []string{ + consts.ComponentKwokController, + consts.ComponentKubeControllerManager, + consts.ComponentKubeScheduler, + } + for _, component := range components { + err := c.StopComponent(ctx, component) + if err != nil { + logger.Error("Failed to stop", err, "component", component) + } + err = c.StartComponent(ctx, component) + if err != nil { + logger.Error("Failed to start", err, "component", component) + } } }() } diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 31b4a9579..9e891f2dd 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -952,7 +952,11 @@ func (c *Cluster) Stop(ctx context.Context) error { return nil } -var importantComponents = map[string]struct{}{ +var startImportantComponents = map[string]struct{}{ + consts.ComponentEtcd: {}, +} + +var stopImportantComponents = map[string]struct{}{ consts.ComponentEtcd: {}, consts.ComponentKubeApiserver: {}, } @@ -961,7 +965,7 @@ var importantComponents = map[string]struct{}{ func (c *Cluster) StartComponent(ctx context.Context, name string) error { logger := log.FromContext(ctx) logger = logger.With("component", name) - if _, important := importantComponents[name]; !important { + if _, important := startImportantComponents[name]; !important { if !c.IsDryRun() { if _, exist, err := c.inspectComponent(ctx, name); err != nil { return err @@ -977,6 +981,9 @@ func (c *Cluster) StartComponent(ctx context.Context, name string) error { if err != nil { return err } + if _, important := startImportantComponents[name]; important { + return nil + } if c.IsDryRun() { return nil } @@ -987,7 +994,7 @@ func (c *Cluster) StartComponent(ctx context.Context, name string) error { func (c *Cluster) StopComponent(ctx context.Context, name string) error { logger := log.FromContext(ctx) logger = logger.With("component", name) - if _, important := importantComponents[name]; !important { + if _, important := stopImportantComponents[name]; !important { if !c.IsDryRun() { if _, exist, err := c.inspectComponent(ctx, name); err != nil { return err @@ -1004,7 +1011,7 @@ func (c *Cluster) StopComponent(ctx context.Context, name string) error { return err } // Once etcd and kube-apiserver are stopped, the cluster will go down - if _, important := importantComponents[name]; important { + if _, important := stopImportantComponents[name]; important { return nil } if c.IsDryRun() { diff --git a/pkg/kwokctl/runtime/kind/cluster_snapshot.go b/pkg/kwokctl/runtime/kind/cluster_snapshot.go index b8a93cbb8..1f6fedda4 100644 --- a/pkg/kwokctl/runtime/kind/cluster_snapshot.go +++ b/pkg/kwokctl/runtime/kind/cluster_snapshot.go @@ -57,20 +57,38 @@ func (c *Cluster) SnapshotSave(ctx context.Context, path string) error { // SnapshotRestore restore the snapshot of cluster func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error { logger := log.FromContext(ctx) - err := c.StopComponent(ctx, consts.ComponentEtcd) - if err != nil { - logger.Error("Failed to stop etcd", err) + clusterName := c.getClusterName() + + components := []string{ + consts.ComponentEtcd, + } + for _, component := range components { + err := c.StopComponent(ctx, component) + if err != nil { + logger.Error("Failed to stop", err, "component", component) + } } defer func() { - err = c.StartComponent(ctx, consts.ComponentEtcd) + for _, component := range components { + err := c.StartComponent(ctx, component) + if err != nil { + logger.Error("Failed to start", err, "component", component) + } + } + + err := c.Stop(ctx) if err != nil { - logger.Error("Failed to start etcd", err) + logger.Error("Failed to stop", err) + } + err = c.Start(ctx) + if err != nil { + logger.Error("Failed to start", err) } }() // Restore snapshot to host temporary directory etcdDataTmp := c.GetWorkdirPath(consts.ComponentEtcd) - err = c.Etcdctl(ctx, "snapshot", "restore", path, "--data-dir", etcdDataTmp) + err := c.Etcdctl(ctx, "snapshot", "restore", path, "--data-dir", etcdDataTmp) if err != nil { return err } @@ -81,9 +99,8 @@ func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error { } }() - kindName := c.getClusterName() // Copy to kind container from host temporary directory - err = c.Exec(ctx, c.runtime, "cp", etcdDataTmp, kindName+":/var/lib/") + err = c.Exec(ctx, c.runtime, "cp", etcdDataTmp, clusterName+":/var/lib/") if err != nil { return err } diff --git a/site/content/en/docs/generated/kwokctl_create_cluster.md b/site/content/en/docs/generated/kwokctl_create_cluster.md index 4c3dada42..8626bbb49 100644 --- a/site/content/en/docs/generated/kwokctl_create_cluster.md +++ b/site/content/en/docs/generated/kwokctl_create_cluster.md @@ -64,6 +64,7 @@ kwokctl create cluster [flags] --kwok-controller-image string Image of kwok-controller, only for docker/podman/nerdctl/kind/kind-podman runtime '${KWOK_IMAGE_PREFIX}/kwok:${KWOK_VERSION}' (default "registry.k8s.io/kwok/kwok:v0.4.0") + --node-lease-duration-seconds uint Duration of node lease in seconds (default 40) --prometheus-binary string Binary of Prometheus, only for binary runtime (default "https://github.com/prometheus/prometheus/releases/download/v2.44.0/prometheus-2.44.0.linux-amd64.tar.gz#prometheus") --prometheus-image string Image of Prometheus, only for docker/podman/nerdctl/kind/kind-podman runtime '${KWOK_PROMETHEUS_IMAGE_PREFIX}/prometheus:${KWOK_PROMETHEUS_VERSION}' diff --git a/test/e2e/helper/node.go b/test/e2e/helper/node.go index 47dde3003..c2fe70c37 100644 --- a/test/e2e/helper/node.go +++ b/test/e2e/helper/node.go @@ -39,6 +39,9 @@ func NewNodeBuilder(name string) *NodeBuilder { "type": "kwok", }, }, + Spec: corev1.NodeSpec{ + PodCIDR: "10.10.0.1/24", + }, }, } } diff --git a/test/e2e/kwokctl/binary/kubectl_test.go b/test/e2e/kwokctl/binary/kubectl_test.go index a84bcfd30..b07162283 100644 --- a/test/e2e/kwokctl/binary/kubectl_test.go +++ b/test/e2e/kwokctl/binary/kubectl_test.go @@ -47,3 +47,15 @@ func TestExec(t *testing.T) { Feature() testEnv.Test(t, f0) } + +func TestRestart(t *testing.T) { + f0 := e2e.CaseRestart(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + +func TestSnapshot(t *testing.T) { + f0 := e2e.CaseSnapshot(kwokctlPath, clusterName, pwd). + Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/docker/kubectl_test.go b/test/e2e/kwokctl/docker/kubectl_test.go index 6a158d9b3..1bce1fc02 100644 --- a/test/e2e/kwokctl/docker/kubectl_test.go +++ b/test/e2e/kwokctl/docker/kubectl_test.go @@ -47,3 +47,15 @@ func TestExec(t *testing.T) { Feature() testEnv.Test(t, f0) } + +func TestRestart(t *testing.T) { + f0 := e2e.CaseRestart(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + +func TestSnapshot(t *testing.T) { + f0 := e2e.CaseSnapshot(kwokctlPath, clusterName, pwd). + Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/kind-podman/kubectl_test.go b/test/e2e/kwokctl/kind-podman/kubectl_test.go index d015167f1..ffd998b54 100644 --- a/test/e2e/kwokctl/kind-podman/kubectl_test.go +++ b/test/e2e/kwokctl/kind-podman/kubectl_test.go @@ -47,3 +47,15 @@ func TestExec(t *testing.T) { Feature() testEnv.Test(t, f0) } + +func TestRestart(t *testing.T) { + f0 := e2e.CaseRestart(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + +func TestSnapshot(t *testing.T) { + f0 := e2e.CaseSnapshot(kwokctlPath, clusterName, pwd). + Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/kind/kubectl_test.go b/test/e2e/kwokctl/kind/kubectl_test.go index 25d76df1c..94d0cb7f3 100644 --- a/test/e2e/kwokctl/kind/kubectl_test.go +++ b/test/e2e/kwokctl/kind/kubectl_test.go @@ -47,3 +47,15 @@ func TestExec(t *testing.T) { Feature() testEnv.Test(t, f0) } + +func TestRestart(t *testing.T) { + f0 := e2e.CaseRestart(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + +func TestSnapshot(t *testing.T) { + f0 := e2e.CaseSnapshot(kwokctlPath, clusterName, pwd). + Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/nerdctl/kubectl_test.go b/test/e2e/kwokctl/nerdctl/kubectl_test.go index a893980a8..790a7a554 100644 --- a/test/e2e/kwokctl/nerdctl/kubectl_test.go +++ b/test/e2e/kwokctl/nerdctl/kubectl_test.go @@ -47,3 +47,15 @@ func TestExec(t *testing.T) { Feature() testEnv.Test(t, f0) } + +func TestRestart(t *testing.T) { + f0 := e2e.CaseRestart(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + +func TestSnapshot(t *testing.T) { + f0 := e2e.CaseSnapshot(kwokctlPath, clusterName, pwd). + Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/podman/kubectl_test.go b/test/e2e/kwokctl/podman/kubectl_test.go index 30a66f65e..bb9bce9a4 100644 --- a/test/e2e/kwokctl/podman/kubectl_test.go +++ b/test/e2e/kwokctl/podman/kubectl_test.go @@ -47,3 +47,15 @@ func TestExec(t *testing.T) { Feature() testEnv.Test(t, f0) } + +func TestRestart(t *testing.T) { + f0 := e2e.CaseRestart(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + +func TestSnapshot(t *testing.T) { + f0 := e2e.CaseSnapshot(kwokctlPath, clusterName, pwd). + Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/restart.go b/test/e2e/restart.go new file mode 100644 index 000000000..7659e95fa --- /dev/null +++ b/test/e2e/restart.go @@ -0,0 +1,73 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "testing" + + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + "sigs.k8s.io/kwok/pkg/utils/exec" + "sigs.k8s.io/kwok/test/e2e/helper" +) + +// CaseRestart is the restart of a test case. +func CaseRestart(kwokctlPath, clusterName string) *features.FeatureBuilder { + node := helper.NewNodeBuilder("node0"). + Build() + pod0 := helper.NewPodBuilder("pod0"). + Build() + pod1 := helper.NewPodBuilder("pod1"). + Build() + + f0 := features.New("Restart"). + Setup(helper.CreateNode(node)). + Setup(helper.CreatePod(pod0)). + Teardown(helper.DeletePod(pod0)). + Teardown(helper.DeleteNode(node)). + Assess("test stop", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + _, err := exec.Command(ctx, kwokctlPath, "stop", "cluster", "--name", clusterName) + if err != nil { + t.Fatal(err) + } + + return ctx + }). + Assess("test start", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + _, err := exec.Command(ctx, kwokctlPath, "start", "cluster", "--name", clusterName) + if err != nil { + t.Fatal(err) + } + + _, err = helper.WaitForAllNodesReady()(ctx, cfg) + if err != nil { + t.Error(err) + } + + _, err = helper.WaitForAllPodsReady()(ctx, cfg) + if err != nil { + t.Error(err) + } + return ctx + }). + Assess("create pod1", helper.CreatePod(pod1)). + Assess("delete pod1", helper.DeletePod(pod1)) + + return f0 +} diff --git a/test/e2e/snapshot.go b/test/e2e/snapshot.go new file mode 100644 index 000000000..a5af76ba0 --- /dev/null +++ b/test/e2e/snapshot.go @@ -0,0 +1,92 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "os" + "testing" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + "sigs.k8s.io/kwok/pkg/utils/exec" + "sigs.k8s.io/kwok/pkg/utils/path" + "sigs.k8s.io/kwok/test/e2e/helper" +) + +// CaseSnapshot is the snapshot of a test case. +func CaseSnapshot(kwokctlPath, clusterName string, tmpDir string) *features.FeatureBuilder { + node := helper.NewNodeBuilder("node0"). + Build() + pod0 := helper.NewPodBuilder("pod0"). + WithNodeName(node.Name). + Build() + + dbPath := path.Join(tmpDir, "snapshot.db") + + return features.New("Snapshot"). + Setup(helper.CreateNode(node)). + Setup(helper.CreatePod(pod0)). + Assess("test snapshot", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + _, err := exec.Command(ctx, kwokctlPath, "snapshot", "save", "--name", clusterName, "--path", dbPath) + if err != nil { + t.Fatal(err) + } + + return ctx + }). + Assess("delete pod", helper.DeletePod(pod0)). + Assess("delete node", helper.DeleteNode(node)). + Assess("test restore", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + _, err := exec.Command(ctx, kwokctlPath, "snapshot", "restore", "--name", clusterName, "--path", dbPath) + if err != nil { + t.Fatal(err) + } + + _ = os.Remove(dbPath) + return ctx + }). + Assess("check restore", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + _, err := helper.WaitForAllNodesReady()(ctx, cfg) + if err != nil { + t.Error(err) + } + + _, err = helper.WaitForAllPodsReady()(ctx, cfg) + if err != nil { + t.Error(err) + } + + client, err := resources.New(cfg.Client().RESTConfig()) + if err != nil { + t.Fatal(err) + } + + var pod corev1.Pod + err = client.Get(ctx, pod0.Name, pod0.Namespace, &pod) + if err != nil { + t.Fatal(err) + } + + return ctx + }). + Assess("delete pod0", helper.DeletePod(pod0)). + Assess("delete node0", helper.DeleteNode(node)) +} diff --git a/test/kwokctl/testdata/binary/create_cluster.txt b/test/kwokctl/testdata/binary/create_cluster.txt index 5fdd726c3..7fec99b19 100644 --- a/test/kwokctl/testdata/binary/create_cluster.txt +++ b/test/kwokctl/testdata/binary/create_cluster.txt @@ -41,5 +41,5 @@ cd /workdir/clusters/ && kube-controller-manager --node- echo $! >/workdir/clusters//pids/kube-controller-manager.pid cd /workdir/clusters/ && kube-scheduler --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & echo $! >/workdir/clusters//pids/kube-scheduler.pid -cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=1200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=40 >/workdir/clusters//logs/kwok-controller.log 2>&1 & echo $! >/workdir/clusters//pids/kwok-controller.pid diff --git a/test/kwokctl/testdata/binary/create_cluster_with_extra.txt b/test/kwokctl/testdata/binary/create_cluster_with_extra.txt index 4c268f92a..4fbe02e43 100644 --- a/test/kwokctl/testdata/binary/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/binary/create_cluster_with_extra.txt @@ -127,7 +127,7 @@ cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-contro echo $! >/workdir/clusters//pids/kube-controller-manager.pid cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-scheduler --v=5 --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & echo $! >/workdir/clusters//pids/kube-scheduler.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kwok-controller --manage-all-nodes=true --v=-4 --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=1200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kwok-controller --manage-all-nodes=true --v=-4 --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=40 >/workdir/clusters//logs/kwok-controller.log 2>&1 & echo $! >/workdir/clusters//pids/kwok-controller.pid cd /workdir/clusters/ && TEST_KEY=TEST_VALUE prometheus --log.level=debug --config.file=/workdir/clusters//prometheus.yaml --web.listen-address=0.0.0.0:9090 >/workdir/clusters//logs/prometheus.log 2>&1 & echo $! >/workdir/clusters//pids/prometheus.pid diff --git a/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt index 3fcb5c627..bc411fd09 100644 --- a/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt @@ -143,7 +143,7 @@ cd /workdir/clusters/ && kube-controller-manager --node- echo $! >/workdir/clusters//pids/kube-controller-manager.pid cd /workdir/clusters/ && kube-scheduler --config=/workdir/clusters//scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32760 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & echo $! >/workdir/clusters//pids/kube-scheduler.pid -cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=1200 --v=DEBUG >/workdir/clusters//logs/kwok-controller.log 2>&1 & +cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=40 --v=DEBUG >/workdir/clusters//logs/kwok-controller.log 2>&1 & echo $! >/workdir/clusters//pids/kwok-controller.pid cd /workdir/clusters/ && prometheus --config.file=/workdir/clusters//prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug >/workdir/clusters//logs/prometheus.log 2>&1 & echo $! >/workdir/clusters//pids/prometheus.pid diff --git a/test/kwokctl/testdata/binary/snapshot_restore_etcd.txt b/test/kwokctl/testdata/binary/snapshot_restore_etcd.txt index 09216c356..835a42181 100644 --- a/test/kwokctl/testdata/binary/snapshot_restore_etcd.txt +++ b/test/kwokctl/testdata/binary/snapshot_restore_etcd.txt @@ -1,7 +1,15 @@ kill $(cat /workdir/clusters//pids/etcd.pid) rm /workdir/clusters//pids/etcd.pid +kill $(cat /workdir/clusters//pids/kube-apiserver.pid) +rm /workdir/clusters//pids/kube-apiserver.pid rm -rf /workdir/clusters//etcd-data # Download https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9--. and extract etcdctl to /workdir/clusters//bin/etcdctl ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:32765 snapshot restore ./snapshot-empty- --data-dir /workdir/clusters//etcd-data rm -rf /workdir/clusters//etcd mv /workdir/clusters//etcd-data /workdir/clusters//etcd +kill $(cat /workdir/clusters//pids/kwok-controller.pid) +rm /workdir/clusters//pids/kwok-controller.pid +kill $(cat /workdir/clusters//pids/kube-controller-manager.pid) +rm /workdir/clusters//pids/kube-controller-manager.pid +kill $(cat /workdir/clusters//pids/kube-scheduler.pid) +rm /workdir/clusters//pids/kube-scheduler.pid diff --git a/test/kwokctl/testdata/binary/start_cluster.txt b/test/kwokctl/testdata/binary/start_cluster.txt index 56183fdef..6574a5b57 100644 --- a/test/kwokctl/testdata/binary/start_cluster.txt +++ b/test/kwokctl/testdata/binary/start_cluster.txt @@ -6,7 +6,7 @@ cd /workdir/clusters/ && kube-controller-manager --node- echo $! >/workdir/clusters//pids/kube-controller-manager.pid cd /workdir/clusters/ && kube-scheduler --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & echo $! >/workdir/clusters//pids/kube-scheduler.pid -cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=1200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=40 >/workdir/clusters//logs/kwok-controller.log 2>&1 & echo $! >/workdir/clusters//pids/kwok-controller.pid cd /workdir/clusters/ && prometheus --config.file=/workdir/clusters//prometheus.yaml --web.listen-address=0.0.0.0:9090 >/workdir/clusters//logs/prometheus.log 2>&1 & echo $! >/workdir/clusters//pids/prometheus.pid diff --git a/test/kwokctl/testdata/docker/create_cluster.txt b/test/kwokctl/testdata/docker/create_cluster.txt index 08b98cb9c..a95380ab4 100644 --- a/test/kwokctl/testdata/docker/create_cluster.txt +++ b/test/kwokctl/testdata/docker/create_cluster.txt @@ -56,7 +56,7 @@ docker create --name=kwok--etcd --pull=never --entrypoint=etcd --n docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 # Add context kwok- to ~/.kube/config docker start kwok--etcd docker start kwok--kube-apiserver diff --git a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt index 3b5342dea..c7148cb3e 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt @@ -143,7 +143,7 @@ docker create --name=kwok--etcd --pull=never --entrypoint=etcd --n docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 docker create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --link=kwok--etcd --link=kwok--kube-apiserver --link=kwok--kube-controller-manager --link=kwok--kube-scheduler --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/extras/prometheus:/extras/tmp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --log.level=debug --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 # Add context kwok- to ~/.kube/config docker start kwok--etcd diff --git a/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt index 23391014d..696fc58f9 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt @@ -160,7 +160,7 @@ docker create --name=kwok--jaeger --pull=never --network=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --link=kwok--jaeger --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.28.0 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 -docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG +docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 --v=DEBUG docker create --name=kwok--dashboard --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 docker create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --link=kwok--etcd --link=kwok--kube-apiserver --link=kwok--kube-controller-manager --link=kwok--kube-scheduler --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config diff --git a/test/kwokctl/testdata/docker/snapshot_restore_etcd.txt b/test/kwokctl/testdata/docker/snapshot_restore_etcd.txt index e308edc88..eb44ab7da 100644 --- a/test/kwokctl/testdata/docker/snapshot_restore_etcd.txt +++ b/test/kwokctl/testdata/docker/snapshot_restore_etcd.txt @@ -1,6 +1,14 @@ # Download https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9--. and extract etcdctl to /workdir/clusters//bin/etcdctl ETCDCTL_API=3 etcdctl snapshot restore ./snapshot-empty- --data-dir /workdir/clusters//etcd-data docker stop kwok--etcd --time=0 +docker stop kwok--kube-apiserver --time=0 docker cp /workdir/clusters//etcd-data kwok--etcd:/ docker start kwok--etcd +docker start kwok--kube-apiserver +docker stop kwok--kwok-controller --time=0 +docker start kwok--kwok-controller +docker stop kwok--kube-controller-manager --time=0 +docker start kwok--kube-controller-manager +docker stop kwok--kube-scheduler --time=0 +docker start kwok--kube-scheduler rm -rf /workdir/clusters//etcd-data diff --git a/test/kwokctl/testdata/kind-podman/snapshot_restore_etcd.txt b/test/kwokctl/testdata/kind-podman/snapshot_restore_etcd.txt index 11c7c2d16..f9ad7c391 100644 --- a/test/kwokctl/testdata/kind-podman/snapshot_restore_etcd.txt +++ b/test/kwokctl/testdata/kind-podman/snapshot_restore_etcd.txt @@ -4,3 +4,5 @@ ETCDCTL_API=3 etcdctl snapshot restore ./snapshot-empty- --data-di podman cp /workdir/clusters//etcd kwok--control-plane:/var/lib/ rm -rf /workdir/clusters//etcd podman exec kwok--control-plane mv /etc/kubernetes/etcd.yaml.bak /etc/kubernetes/manifests/etcd.yaml +podman stop kwok--control-plane +podman start kwok--control-plane diff --git a/test/kwokctl/testdata/kind/snapshot_restore_etcd.txt b/test/kwokctl/testdata/kind/snapshot_restore_etcd.txt index efd228f9c..64766c99c 100644 --- a/test/kwokctl/testdata/kind/snapshot_restore_etcd.txt +++ b/test/kwokctl/testdata/kind/snapshot_restore_etcd.txt @@ -4,3 +4,5 @@ ETCDCTL_API=3 etcdctl snapshot restore ./snapshot-empty- --data-di docker cp /workdir/clusters//etcd kwok--control-plane:/var/lib/ rm -rf /workdir/clusters//etcd docker exec kwok--control-plane mv /etc/kubernetes/etcd.yaml.bak /etc/kubernetes/manifests/etcd.yaml +docker stop kwok--control-plane +docker start kwok--control-plane diff --git a/test/kwokctl/testdata/nerdctl/create_cluster.txt b/test/kwokctl/testdata/nerdctl/create_cluster.txt index cacd5beb4..505c28fd2 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster.txt @@ -56,7 +56,7 @@ nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd -- nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 # Add context kwok- to ~/.kube/config nerdctl start kwok--etcd nerdctl start kwok--kube-apiserver diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt index 76f94cf56..a4f05a9c2 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt @@ -143,7 +143,7 @@ nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd -- nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 nerdctl create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/extras/prometheus:/extras/tmp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --log.level=debug --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 # Add context kwok- to ~/.kube/config nerdctl start kwok--etcd diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt index 837920ff8..776081eed 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt @@ -160,7 +160,7 @@ nerdctl create --name=kwok--jaeger --pull=never --network=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.28.0 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 -nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG +nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 --v=DEBUG nerdctl create --name=kwok--dashboard --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 nerdctl create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config diff --git a/test/kwokctl/testdata/nerdctl/snapshot_restore_etcd.txt b/test/kwokctl/testdata/nerdctl/snapshot_restore_etcd.txt index a06e0ce96..12b72cc41 100644 --- a/test/kwokctl/testdata/nerdctl/snapshot_restore_etcd.txt +++ b/test/kwokctl/testdata/nerdctl/snapshot_restore_etcd.txt @@ -5,4 +5,11 @@ nerdctl cp /workdir/clusters//etcd-data kwok--etcd --time=0 nerdctl start kwok--etcd nerdctl start kwok--kube-apiserver +nerdctl stop kwok--kwok-controller --time=0 +nerdctl start kwok--kwok-controller +nerdctl stop kwok--kube-controller-manager --time=0 +nerdctl start kwok--kube-controller-manager +nerdctl stop kwok--kube-scheduler --time=0 +nerdctl start kwok--kube-scheduler +nerdctl start kwok--kube-apiserver rm -rf /workdir/clusters//etcd-data diff --git a/test/kwokctl/testdata/podman/create_cluster.txt b/test/kwokctl/testdata/podman/create_cluster.txt index 24705c293..c30ffab6a 100644 --- a/test/kwokctl/testdata/podman/create_cluster.txt +++ b/test/kwokctl/testdata/podman/create_cluster.txt @@ -56,7 +56,7 @@ podman create --name=kwok--etcd --pull=never --entrypoint=etcd --n podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 # Add context kwok- to ~/.kube/config podman start kwok--etcd podman start kwok--kube-apiserver diff --git a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt index 23ddd4c3d..f3329a2c3 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt @@ -143,7 +143,7 @@ podman create --name=kwok--etcd --pull=never --entrypoint=etcd --n podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 podman create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --requires=kwok--etcd --requires=kwok--kube-apiserver --requires=kwok--kube-controller-manager --requires=kwok--kube-scheduler --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/extras/prometheus:/extras/tmp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --log.level=debug --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 # Add context kwok- to ~/.kube/config podman start kwok--etcd diff --git a/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt index 4c48f92a8..0ac47e6c4 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt @@ -160,7 +160,7 @@ podman create --name=kwok--jaeger --pull=never --network=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --requires=kwok--jaeger --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.28.0 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 -podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG +podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=40 --v=DEBUG podman create --name=kwok--dashboard --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 podman create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --requires=kwok--etcd --requires=kwok--kube-apiserver --requires=kwok--kube-controller-manager --requires=kwok--kube-scheduler --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config diff --git a/test/kwokctl/testdata/podman/snapshot_restore_etcd.txt b/test/kwokctl/testdata/podman/snapshot_restore_etcd.txt index 9395b4153..5cc902213 100644 --- a/test/kwokctl/testdata/podman/snapshot_restore_etcd.txt +++ b/test/kwokctl/testdata/podman/snapshot_restore_etcd.txt @@ -1,6 +1,14 @@ # Download https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9--. and extract etcdctl to /workdir/clusters//bin/etcdctl ETCDCTL_API=3 etcdctl snapshot restore ./snapshot-empty- --data-dir /workdir/clusters//etcd-data podman stop kwok--etcd --time=0 +podman stop kwok--kube-apiserver --time=0 podman cp /workdir/clusters//etcd-data kwok--etcd:/ podman start kwok--etcd +podman start kwok--kube-apiserver +podman stop kwok--kwok-controller --time=0 +podman start kwok--kwok-controller +podman stop kwok--kube-controller-manager --time=0 +podman start kwok--kube-controller-manager +podman stop kwok--kube-scheduler --time=0 +podman start kwok--kube-scheduler rm -rf /workdir/clusters//etcd-data