Skip to content

Commit

Permalink
Cleanup after ephemeral volume test (#127)
Browse files Browse the repository at this point in the history
* Made no-cleanup flag available only to the eph functional test, since it's NA to other tests. Auto-cleanup is now the default for the eph volume test, other tests never clean up.

* Added a unit-test to cover the fixed scenario (proper no-cleanup handling for ephemeral volume test).

* Linter errors fixed.

* Fix missing kubeconfig in GH action.

* Fix missing kubeconfig in GH action.

* Fixed typos.

---------

Co-authored-by: alikdell <[email protected]>
  • Loading branch information
alexemc and alikdell authored Jan 15, 2025
1 parent 4f1528a commit dd44b6c
Show file tree
Hide file tree
Showing 6 changed files with 256 additions and 38 deletions.
50 changes: 25 additions & 25 deletions pkg/cmd/functionalcmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,6 @@ func GetFunctionalTestCommand() cli.Command {
Usage: "set the timeout value for all of the resources (accepts format like 2h30m15s) default is 0s",
Value: "0s",
},
cli.BoolFlag{
Name: "no-cleanup, nc",
Usage: "include this flag do disable cleanup between iterations",
},
cli.BoolFlag{
Name: "no-cleanup-on-fail, ncof",
Usage: "include this flag do disable cleanup on fail",
},
cli.StringFlag{
Name: "description, de",
Usage: "To provide test case description",
Expand Down Expand Up @@ -109,7 +101,7 @@ func GetFunctionalTestCommand() cli.Command {
return funtestCmd
}

func createFunctionalSuiteRunner(c *cli.Context) *runner.FunctionalSuiteRunner {
func createFunctionalSuiteRunner(c *cli.Context, noCleanup, noCleanupOnFail bool) *runner.FunctionalSuiteRunner {
// Parse timeout
timeout, err := time.ParseDuration(c.String("timeout"))
if err != nil {
Expand Down Expand Up @@ -139,16 +131,16 @@ func createFunctionalSuiteRunner(c *cli.Context) *runner.FunctionalSuiteRunner {
// enable logging for each suite in a separate file.
logFile, err := os.OpenFile(filepath.Clean(path), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o600)
if err != nil {
log.Errorf("Error creating log file:%s", err)
log.Errorf("Error creating log file: %v", err)
}
log.SetOutput(io.MultiWriter(os.Stdout, logFile))

return runner.NewFunctionalSuiteRunner(
c.String("config"),
c.String("namespace"),
timeOutInSeconds,
c.Bool("no-cleanup"),
c.Bool("no-cleanup-on-fail"),
noCleanup,
noCleanupOnFail,
c.Bool("no-reports"),
scDB,
)
Expand All @@ -175,7 +167,7 @@ func getVolumeDeletionCommand(globalFlags []cli.Flag) cli.Command {
globalFlags...,
),
Action: func(c *cli.Context) error {
sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
pvcName := c.String("pvc-name")
pvcNamespace := c.String("pvc-namespace")
desc := c.String("description")
Expand Down Expand Up @@ -215,7 +207,7 @@ func getPodDeletionCommand(globalFlags []cli.Flag) cli.Command {
globalFlags...,
),
Action: func(c *cli.Context) error {
sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
podName := c.String("pod-name")
podNamespace := c.String("pod-namespace")
desc := c.String("description")
Expand Down Expand Up @@ -260,7 +252,7 @@ func getCloneVolumeDeletionCommand(globalFlags []cli.Flag) cli.Command {
globalFlags...,
),
Action: func(c *cli.Context) error {
sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
volName := c.String("clone-volume-name")
podName := c.String("clone-pod-name")
namespace := c.String("resource-namespace")
Expand Down Expand Up @@ -303,7 +295,7 @@ func getFunctionalSnapDeletionCommand(globalFlags []cli.Flag) cli.Command {
globalFlags...,
),
Action: func(c *cli.Context) error {
sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
snapName := c.String("volume-snapshot-name")
namespace := c.String("resource-namespace")
desc := c.String("description")
Expand Down Expand Up @@ -376,7 +368,7 @@ func getFunctionalVolumeCreateCommand(globalFlags []cli.Flag) cli.Command {
RawBlock: blockVol,
},
}
sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
sr.RunFunctionalSuites(s)

return nil
Expand Down Expand Up @@ -442,7 +434,7 @@ func getFunctionalCloneVolumeCommand(globalFlags []cli.Flag) cli.Command {
},
}

sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
sr.RunFunctionalSuites(s)

return nil
Expand Down Expand Up @@ -514,7 +506,7 @@ func getFunctionalProvisioningCommand(globalFlags []cli.Flag) cli.Command {
},
}

sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
sr.RunFunctionalSuites(s)

return nil
Expand Down Expand Up @@ -588,7 +580,7 @@ func getFunctionalSnapCreationCommand(globalFlags []cli.Flag) cli.Command {
},
}

sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
sr.RunFunctionalSuites(s)

return nil
Expand Down Expand Up @@ -644,7 +636,7 @@ func getFunctionalMultiAttachVolCommand(globalFlags []cli.Flag) cli.Command {
},
}

sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
sr.RunFunctionalSuites(s)

return nil
Expand Down Expand Up @@ -682,6 +674,14 @@ func getFunctionalEphemeralCreationCommand(globalFlags []cli.Flag) cli.Command {
Name: "pod-name, pname",
Usage: "custom name for pod and cloned volume pod to create",
},
cli.BoolFlag{
Name: "no-cleanup, nc",
Usage: "include this flag to disable cleanup after test",
},
cli.BoolFlag{
Name: "no-cleanup-on-fail, ncof",
Usage: "include this flag to disable cleanup on fail",
},
},
globalFlags...,
),
Expand Down Expand Up @@ -715,7 +715,7 @@ func getFunctionalEphemeralCreationCommand(globalFlags []cli.Flag) cli.Command {
},
}

sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, c.Bool("no-cleanup"), c.Bool("no-cleanup-on-fail"))
sr.RunFunctionalSuites(s)

return nil
Expand Down Expand Up @@ -790,7 +790,7 @@ func getNodeDrainCommand(globalFlags []cli.Flag) cli.Command {
globalFlags...,
),
Action: func(c *cli.Context) error {
sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
nodeName := c.String("node-name")
nodeNamespace := c.String("node-namespace")
gracePeriodSeconds := c.Int("grace-period-seconds")
Expand Down Expand Up @@ -826,7 +826,7 @@ func getNodeUnCordonCommand(globalFlags []cli.Flag) cli.Command {
globalFlags...,
),
Action: func(c *cli.Context) error {
sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
nodeName := c.String("node-name")
desc := c.String("description")

Expand Down Expand Up @@ -891,7 +891,7 @@ func getCapacityTrackingCommand(globalFlags []cli.Flag) cli.Command {
},
}

sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, true, true)
sr.RunFunctionalSuites(s)

return nil
Expand Down
199 changes: 199 additions & 0 deletions pkg/cmd/functionalcmd_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
package cmd

import (
"context"
"flag"
"fmt"
"os"
"strings"
"testing"

"github.com/stretchr/testify/assert"

"github.com/dell/cert-csi/pkg/k8sclient"
"github.com/dell/cert-csi/pkg/observer"
"github.com/dell/cert-csi/pkg/testcore/suites"
"github.com/urfave/cli"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/version"
"k8s.io/client-go/discovery/fake"
"k8s.io/client-go/kubernetes"
fakeClient "k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/rest"
clienttesting "k8s.io/client-go/testing"
)

type MockCleanupTestSuite struct {
t *testing.T
}

func (m *MockCleanupTestSuite) GetName() string {
return "mock"
}

func (m *MockCleanupTestSuite) Run(_ context.Context, _ string, _ *k8sclient.Clients) (delFunc func() error, e error) {
m.t.Logf("mock test suite run")
return nil, nil
}

func (m *MockCleanupTestSuite) GetObservers(_ observer.Type) []observer.Interface {
return nil
}

func (m *MockCleanupTestSuite) GetClients(_ string, _ *k8sclient.KubeClient) (*k8sclient.Clients, error) {
return nil, nil
}

func (m *MockCleanupTestSuite) GetNamespace() string {
return "mock-ns-prefix"
}

func (m *MockCleanupTestSuite) Parameters() string {
return ""
}

type clientTestContext struct {
testNamespace string
namespaceDeleted bool
t *testing.T
}

func TestCleanupAfterTest(t *testing.T) {
// Create a temporary file that contains a simple kubernetes config
// and set the environment variable to point to it
confPath, err := createDummyKubeConfig(t.TempDir(), t)
assert.NoError(t, err)

s := []suites.Interface{
&MockCleanupTestSuite{
t: t,
},
}

FuncNewClientSetOriginal := k8sclient.FuncNewClientSet
defer func() {
k8sclient.FuncNewClientSet = FuncNewClientSetOriginal
}()

clientCtx := &clientTestContext{t: t}

k8sclient.FuncNewClientSet = func(_ *rest.Config) (kubernetes.Interface, error) {
return createFakeKubeClient(clientCtx)
}

fset := flag.NewFlagSet("unit-test", flag.ContinueOnError)
timeoutFlag := &cli.StringFlag{
Name: "timeout",
Value: "1m",
}
timeoutFlag.Apply(fset)
configFlag := &cli.StringFlag{
Name: "config",
Value: confPath,
}
configFlag.Apply(fset)

c := cli.NewContext(nil, fset, nil)

tests := []struct {
name string
noCleanup bool
expectCleanup bool
}{
{
name: "Functional test with no-cleanup = false",
noCleanup: false,
expectCleanup: true,
},
{
name: "Functional test with no-cleanup = true",
noCleanup: true,
expectCleanup: false,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Reset callbacks state before each test
clientCtx.testNamespace = ""
clientCtx.namespaceDeleted = false

sr := createFunctionalSuiteRunner(c, tt.noCleanup, tt.noCleanup)
sr.RunFunctionalSuites(s)

if clientCtx.namespaceDeleted != tt.expectCleanup {
t.Errorf("Expected test namespace deletion %v, but got %v", tt.expectCleanup, clientCtx.namespaceDeleted)
}
})
}
}

func createFakeKubeClient(ctx *clientTestContext) (kubernetes.Interface, error) {
client := fakeClient.NewSimpleClientset()
client.Discovery().(*fake.FakeDiscovery).FakedServerVersion = &version.Info{
Major: "1",
Minor: "32",
GitVersion: "v1.32.0",
}
client.Fake.PrependReactor("create", "namespaces", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
createAction := action.(clienttesting.CreateAction)
namespace := createAction.GetObject().(*corev1.Namespace)
if strings.HasPrefix(namespace.Name, "mock-ns-prefix-") {
ctx.t.Logf("namespace %s creation called", namespace.Name)
if ctx.testNamespace == "" {
ctx.testNamespace = namespace.Name
} else {
return true, nil, fmt.Errorf("repeated test namespace creation call: was %s, now %s", ctx.testNamespace, namespace.Name)
}
return true, namespace, nil
}
return true, nil, fmt.Errorf("unexpected namespace creation %s", namespace.Name)
})
client.Fake.PrependReactor("delete", "namespaces", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
deleteAction := action.(clienttesting.DeleteAction)
if ctx.testNamespace != "" && deleteAction.GetName() == ctx.testNamespace {
ctx.t.Logf("namespace %s deletion called", deleteAction.GetName())
ctx.namespaceDeleted = true
return true, nil, nil
}
return true, nil, fmt.Errorf("unexpected namespace deletion %s", deleteAction.GetName())
})
return client, nil
}

func createDummyKubeConfig(tmpDir string, t *testing.T) (string, error) {
// Define a simple kube client config
kubeConfig := `
apiVersion: v1
kind: Config
clusters:
- cluster:
server: https://unit.test
name: test-cluster
contexts:
- context:
cluster: test-cluster
user: test-user
name: test-context
current-context: test-context
users:
- name: test-user
user:
token: test-token
`

confPath := tmpDir + "/kube.config"

err := os.WriteFile(confPath, []byte(kubeConfig), 0o600)
if err != nil {
return "", fmt.Errorf("failed to write dummy kube config file: %v", err)
}

// Set the environment variable to point to the temporary file
t.Setenv("KUBECONFIG", confPath)

// Print the path to the temporary file
t.Logf("Created dummy kube config file at: %s", confPath)
return confPath, nil
}
2 changes: 1 addition & 1 deletion pkg/cmd/testcmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -1085,7 +1085,7 @@ func getEphemeralCreationCommand(globalFlags []cli.Flag) cli.Command {
},
}

sr := createFunctionalSuiteRunner(c)
sr := createFunctionalSuiteRunner(c, c.Bool("no-cleanup"), c.Bool("no-cleanup-on-fail"))
sr.RunFunctionalSuites(s)

return nil
Expand Down
Loading

0 comments on commit dd44b6c

Please sign in to comment.