diff --git a/pkg/migration/filesystem.go b/pkg/migration/filesystem.go new file mode 100644 index 00000000..9c9b68c1 --- /dev/null +++ b/pkg/migration/filesystem.go @@ -0,0 +1,151 @@ +package migration + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + + "github.com/pkg/errors" + "github.com/spf13/afero" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/yaml" + sigsyaml "sigs.k8s.io/yaml" +) + +// FileSystemSource is a source implementation to read resources from filesystem +type FileSystemSource struct { + index int + items []UnstructuredWithMetadata + afero afero.Afero +} + +// FileSystemSourceOption allows you to configure FileSystemSource +type FileSystemSourceOption func(*FileSystemSource) + +// FsWithFileSystem configures the filesystem to use. Used mostly for testing. +func FsWithFileSystem(f afero.Fs) FileSystemSourceOption { + return func(fs *FileSystemSource) { + fs.afero = afero.Afero{Fs: f} + } +} + +// NewFileSystemSource returns a FileSystemSource +func NewFileSystemSource(dir string, opts ...FileSystemSourceOption) (*FileSystemSource, error) { + fs := &FileSystemSource{ + afero: afero.Afero{Fs: afero.NewOsFs()}, + } + for _, f := range opts { + f(fs) + } + + if err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return errors.Wrap(err, fmt.Sprintf("cannot read %s", path)) + } + + if info.IsDir() { + return nil + } + + data, err := fs.afero.ReadFile(path) + if err != nil { + return errors.Wrap(err, "cannot read source file") + } + + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewBufferString(string(data)), 1024) + u := &unstructured.Unstructured{} + if err := decoder.Decode(&u); err != nil { + return errors.Wrap(err, "cannot decode read data") + } + + fs.items = append(fs.items, UnstructuredWithMetadata{ + Object: *u, + Metadata: Metadata{ + Path: path, + }, + }) + + return nil + }); err != nil { + return nil, errors.Wrap(err, "cannot read source directory") + } + + return fs, nil +} + +// HasNext checks the next item +func (fs *FileSystemSource) HasNext() (bool, error) { + return fs.index < len(fs.items), nil +} + +// Next returns the next item of slice +func (fs *FileSystemSource) Next() (UnstructuredWithMetadata, error) { + if hasNext, _ := fs.HasNext(); hasNext { + item := fs.items[fs.index] + fs.index++ + return item, nil + } + return UnstructuredWithMetadata{}, errors.New("no more elements") +} + +// FileSystemTarget is a target implementation to write/patch/delete resources to file system +type FileSystemTarget struct { + afero afero.Afero +} + +// FileSystemTargetOption allows you to configure FileSystemTarget +type FileSystemTargetOption func(*FileSystemTarget) + +// FtWithFileSystem configures the filesystem to use. Used mostly for testing. +func FtWithFileSystem(f afero.Fs) FileSystemTargetOption { + return func(ft *FileSystemTarget) { + ft.afero = afero.Afero{Fs: f} + } +} + +// NewFileSystemTarget returns a FileSystemTarget +func NewFileSystemTarget(opts ...FileSystemTargetOption) *FileSystemTarget { + ft := &FileSystemTarget{ + afero: afero.Afero{Fs: afero.NewOsFs()}, + } + for _, f := range opts { + f(ft) + } + return ft +} + +// Put writes input to filesystem +func (ft *FileSystemTarget) Put(o UnstructuredWithMetadata) error { + b, err := sigsyaml.Marshal(o.Object.Object) + if err != nil { + return errors.Wrap(err, "cannot marshal object") + } + if o.Metadata.Parents != "" { + f, err := ft.afero.OpenFile(o.Metadata.Path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return errors.Wrap(err, "cannot open file") + } + + defer f.Close() //nolint:errcheck + + if _, err = f.WriteString(fmt.Sprintf("\n---\n\n%s", string(b))); err != nil { + return errors.Wrap(err, "cannot write file") + } + } else { + f, err := ft.afero.Create(o.Metadata.Path) + if err != nil { + return errors.Wrap(err, "cannot create file") + } + if _, err := f.Write(b); err != nil { + return errors.Wrap(err, "cannot write file") + } + } + + return nil +} + +// Delete deletes a file from filesystem +func (ft *FileSystemTarget) Delete(o UnstructuredWithMetadata) error { + return ft.afero.Remove(o.Metadata.Path) +} diff --git a/pkg/migration/filesystem_test.go b/pkg/migration/filesystem_test.go new file mode 100644 index 00000000..8cf452e8 --- /dev/null +++ b/pkg/migration/filesystem_test.go @@ -0,0 +1,256 @@ +package migration + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/pkg/errors" + "github.com/spf13/afero" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +var ( + unstructuredAwsVpc = map[string]interface{}{ + "apiVersion": "ec2.aws.crossplane.io/v1beta1", + "kind": "VPC", + "metadata": map[string]interface{}{ + "name": "sample-vpc", + }, + "spec": map[string]interface{}{ + "forProvider": map[string]interface{}{ + "region": "us-west-1", + "cidrBlock": "172.16.0.0/16", + }, + }, + } + + unstructuredResourceGroup = map[string]interface{}{ + "apiVersion": "azure.crossplane.io/v1beta1", + "kind": "ResourceGroup", + "metadata": map[string]interface{}{ + "name": "example-resources", + }, + "spec": map[string]interface{}{ + "forProvider": map[string]interface{}{ + "location": "West Europe", + }, + }, + } +) + +func TestNewFileSystemSource(t *testing.T) { + type args struct { + dir string + a func() afero.Afero + } + type want struct { + fs *FileSystemSource + err error + } + + cases := map[string]struct { + args + want + }{ + "Successful": { + args: args{ + dir: "testdata", + a: func() afero.Afero { + fss := afero.Afero{Fs: afero.NewMemMapFs()} + _ = fss.WriteFile("testdata/source/awsvpc.yaml", + []byte("apiVersion: ec2.aws.crossplane.io/v1beta1\nkind: VPC\nmetadata:\n name: sample-vpc\nspec:\n forProvider:\n cidrBlock: 172.16.0.0/16\n region: us-west-1\n"), + 0600) + _ = fss.WriteFile("testdata/source/resourcegroup.yaml", + []byte("apiVersion: azure.crossplane.io/v1beta1\nkind: ResourceGroup\nmetadata:\n name: example-resources\nspec:\n forProvider:\n location: West Europe\n"), + 0600) + return fss + }, + }, + want: want{ + fs: &FileSystemSource{ + index: 0, + items: []UnstructuredWithMetadata{ + { + Object: unstructured.Unstructured{ + Object: unstructuredAwsVpc, + }, + Metadata: Metadata{ + Path: "testdata/source/awsvpc.yaml", + }, + }, + { + Object: unstructured.Unstructured{ + Object: unstructuredResourceGroup, + }, + Metadata: Metadata{ + Path: "testdata/source/resourcegroup.yaml", + }, + }, + }, + }, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + files := tc.args.a() + fs, err := NewFileSystemSource("testdata/source", FsWithFileSystem(files)) + if err != nil { + t.Fatalf("Failed to initialize a new FileSystemSource: %v", err) + } + if diff := cmp.Diff(tc.want.err, err); diff != "" { + t.Errorf("\nNext(...): -want, +got:\n%s", diff) + } + if diff := cmp.Diff(tc.want.fs.items, fs.items); diff != "" { + t.Errorf("\nNext(...): -want, +got:\n%s", diff) + } + }) + } +} + +func TestFileSystemTarget_Put(t *testing.T) { + type args struct { + o UnstructuredWithMetadata + a func() afero.Afero + } + type want struct { + data string + err error + } + + cases := map[string]struct { + args + want + }{ + "Write": { + args: args{ + o: UnstructuredWithMetadata{ + Object: unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "ec2.aws.upbound.io/v1beta1", + "kind": "VPC", + "metadata": map[string]interface{}{ + "name": "sample-vpc", + }, + "spec": map[string]interface{}{ + "forProvider": map[string]interface{}{ + "region": "us-west-1", + "cidrBlock": "172.16.0.0/16", + }, + }, + }, + }, + Metadata: Metadata{ + Path: "testdata/source/awsvpc.yaml", + }, + }, + a: func() afero.Afero { + return afero.Afero{Fs: afero.NewMemMapFs()} + }, + }, + want: want{ + data: "apiVersion: ec2.aws.upbound.io/v1beta1\nkind: VPC\nmetadata:\n name: sample-vpc\nspec:\n forProvider:\n cidrBlock: 172.16.0.0/16\n region: us-west-1\n", + err: nil, + }, + }, + "Append": { + args: args{ + o: UnstructuredWithMetadata{ + Object: unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "azure.crossplane.io/v1beta1", + "kind": "ResourceGroup", + "metadata": map[string]interface{}{ + "name": "example-resources", + }, + "spec": map[string]interface{}{ + "forProvider": map[string]interface{}{ + "location": "West Europe", + }, + }, + }, + }, + Metadata: Metadata{ + Path: "testdata/source/awsvpc.yaml", + Parents: "parent metadata", + }, + }, + a: func() afero.Afero { + fss := afero.Afero{Fs: afero.NewMemMapFs()} + _ = fss.WriteFile("testdata/source/awsvpc.yaml", + []byte("apiVersion: ec2.aws.upbound.io/v1beta1\nkind: VPC\nmetadata:\n name: sample-vpc\nspec:\n forProvider:\n cidrBlock: 172.16.0.0/16\n region: us-west-1\n"), + 0600) + return fss + }, + }, + want: want{ + data: "apiVersion: ec2.aws.upbound.io/v1beta1\nkind: VPC\nmetadata:\n name: sample-vpc\nspec:\n forProvider:\n cidrBlock: 172.16.0.0/16\n region: us-west-1\n\n---\n\napiVersion: azure.crossplane.io/v1beta1\nkind: ResourceGroup\nmetadata:\n name: example-resources\nspec:\n forProvider:\n location: West Europe\n", + err: nil, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + files := tc.args.a() + ft := NewFileSystemTarget(FtWithFileSystem(files)) + if err := ft.Put(tc.args.o); err != nil { + t.Error(err) + } + b, err := ft.afero.ReadFile("testdata/source/awsvpc.yaml") + if diff := cmp.Diff(tc.want.err, err); diff != "" { + t.Errorf("\nNext(...): -want, +got:\n%s", diff) + } + if diff := cmp.Diff(tc.want.data, string(b)); diff != "" { + t.Errorf("\nNext(...): -want, +got:\n%s", diff) + } + }) + } +} + +func TestFileSystemTarget_Delete(t *testing.T) { + type args struct { + o UnstructuredWithMetadata + a func() afero.Afero + } + type want struct { + err error + } + cases := map[string]struct { + args + want + }{ + "Successful": { + args: args{ + o: UnstructuredWithMetadata{ + Metadata: Metadata{ + Path: "testdata/source/awsvpc.yaml", + }, + }, + a: func() afero.Afero { + fss := afero.Afero{Fs: afero.NewMemMapFs()} + _ = fss.WriteFile("testdata/source/awsvpc.yaml", + []byte("apiVersion: ec2.aws.upbound.io/v1beta1\nkind: VPC\nmetadata:\n name: sample-vpc\nspec:\n forProvider:\n cidrBlock: 172.16.0.0/16\n region: us-west-1\n"), + 0600) + return fss + }, + }, + want: want{ + err: errors.New(fmt.Sprintf("%s: %s", "open testdata/source/awsvpc.yaml", afero.ErrFileNotFound)), + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + files := tc.args.a() + ft := NewFileSystemTarget(FtWithFileSystem(files)) + if err := ft.Delete(tc.args.o); err != nil { + t.Error(err) + } + _, err := ft.afero.ReadFile("testdata/source/awsvpc.yaml") + if diff := cmp.Diff(tc.want.err.Error(), err.Error()); diff != "" { + t.Errorf("\nNext(...): -want, +got:\n%s", diff) + } + }) + } +} diff --git a/pkg/migration/kubernetes.go b/pkg/migration/kubernetes.go new file mode 100644 index 00000000..1f973d61 --- /dev/null +++ b/pkg/migration/kubernetes.go @@ -0,0 +1,84 @@ +package migration + +import ( + "context" + "strings" + + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/tools/clientcmd" +) + +// KubernetesSource is a source implementation to read resources from Kubernetes +// cluster. +type KubernetesSource struct { + index int + items []UnstructuredWithMetadata + dynamicClient dynamic.Interface +} + +// NewKubernetesSource returns a KubernetesSource +// DynamicClient is used here to query resources. +// Elements of gvks (slice of GroupVersionKind) are passed to the Dynamic Client +// in a loop to get list of resources. +// An example element of gvks slice: +// Group: "ec2.aws.upbound.io", +// Version: "v1beta1", +// Kind: "VPC", +func NewKubernetesSource(dynamicClient dynamic.Interface, gvks []schema.GroupVersionKind) (*KubernetesSource, error) { + ks := &KubernetesSource{ + dynamicClient: dynamicClient, + } + for _, gvk := range gvks { + ri := dynamicClient.Resource( + schema.GroupVersionResource{ + Group: gvk.Group, + Version: gvk.Version, + // we need to add plural appendix to end of kind name + Resource: strings.ToLower(gvk.Kind) + "s", + }) + unstructuredList, err := ri.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, errors.Wrap(err, "cannot list resources") + } + for _, u := range unstructuredList.Items { + ks.items = append(ks.items, UnstructuredWithMetadata{ + Object: u, + Metadata: Metadata{ + Path: string(u.GetUID()), + }, + }) + } + } + return ks, nil +} + +// HasNext checks the next item +func (ks *KubernetesSource) HasNext() (bool, error) { + return ks.index < len(ks.items), nil +} + +// Next returns the next item of slice +func (ks *KubernetesSource) Next() (UnstructuredWithMetadata, error) { + if hasNext, _ := ks.HasNext(); hasNext { + item := ks.items[ks.index] + ks.index++ + return item, nil + } + return UnstructuredWithMetadata{}, errors.New("no more elements") +} + +// InitializeDynamicClient returns a dynamic client +func InitializeDynamicClient(kubeconfigPath string) (dynamic.Interface, error) { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return nil, errors.Wrap(err, "cannot create rest config object") + } + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, errors.Wrap(err, "cannot initialize dynamic client") + } + return dynamicClient, nil +} diff --git a/pkg/migration/kubernetes_test.go b/pkg/migration/kubernetes_test.go new file mode 100644 index 00000000..2fe42682 --- /dev/null +++ b/pkg/migration/kubernetes_test.go @@ -0,0 +1,63 @@ +package migration + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic/fake" +) + +func TestNewKubernetesSource(t *testing.T) { + type args struct { + gvks []schema.GroupVersionKind + } + type want struct { + ks *KubernetesSource + err error + } + + cases := map[string]struct { + args + want + }{ + "Successful": { + args: args{ + gvks: []schema.GroupVersionKind{ + { + Group: "ec2.aws.crossplane.io", + Version: "v1beta1", + Kind: "VPC", + }, + }, + }, + want: want{ + ks: &KubernetesSource{ + items: []UnstructuredWithMetadata{ + { + Object: unstructured.Unstructured{ + Object: unstructuredAwsVpc, + }, + }, + }, + }, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + dynamicClient := fake.NewSimpleDynamicClient(runtime.NewScheme(), + &unstructured.Unstructured{Object: unstructuredAwsVpc}, + &unstructured.Unstructured{Object: unstructuredResourceGroup}) + ks, err := NewKubernetesSource(dynamicClient, tc.args.gvks) + if diff := cmp.Diff(tc.want.err, err); diff != "" { + t.Errorf("\nNext(...): -want, +got:\n%s", diff) + } + if diff := cmp.Diff(tc.want.ks.items, ks.items); diff != "" { + t.Errorf("\nNext(...): -want, +got:\n%s", diff) + } + }) + } +} diff --git a/pkg/migration/testdata/source/awsvpc.yaml b/pkg/migration/testdata/source/awsvpc.yaml new file mode 100644 index 00000000..c62e492a --- /dev/null +++ b/pkg/migration/testdata/source/awsvpc.yaml @@ -0,0 +1,8 @@ +apiVersion: ec2.aws.crossplane.io/v1beta1 +kind: VPC +metadata: + name: sample-vpc +spec: + forProvider: + region: us-west-1 + cidrBlock: 172.16.0.0/16 diff --git a/pkg/migration/testdata/source/resourcegroup.yaml b/pkg/migration/testdata/source/resourcegroup.yaml new file mode 100644 index 00000000..a84ff84a --- /dev/null +++ b/pkg/migration/testdata/source/resourcegroup.yaml @@ -0,0 +1,7 @@ +apiVersion: azure.crossplane.io/v1beta1 +kind: ResourceGroup +metadata: + name: example-resources +spec: + forProvider: + location: "West Europe"