From 0ab1b2b7023be8f0667656e0ed8e21925df175eb Mon Sep 17 00:00:00 2001 From: zongz Date: Mon, 9 Sep 2024 10:16:24 +0800 Subject: [PATCH 1/6] feat: make 'kpm run' supports mvs Signed-off-by: zongz --- pkg/client/client.go | 10 +++ pkg/client/client_test.go | 56 ++++++++++--- pkg/client/resolver.go | 5 ++ pkg/client/run_test.go | 82 +++++++++++++++++++ .../resolve_deps/my_kcl_compile/kcl.mod | 5 ++ .../resolve_deps/my_kcl_compile/kcl.mod.lock | 6 ++ .../resolve_metadata/with_package/kcl.mod | 2 +- .../with_package/kcl.mod.lock | 14 ++-- .../test_data/test_run_git_package/kcl.mod | 2 +- .../test_data/test_run_git_package/main.k | 11 +-- .../test_run_with_mvs/update_0/dep_0/kcl.mod | 7 ++ .../update_0/dep_0/kcl.mod.lock | 6 ++ .../test_run_with_mvs/update_0/dep_0/main.k | 1 + .../test_run_with_mvs/update_0/dep_1/kcl.mod | 7 ++ .../update_0/dep_1/kcl.mod.lock | 6 ++ .../test_run_with_mvs/update_0/dep_1/main.k | 1 + .../test_run_with_mvs/update_0/dep_2/kcl.mod | 7 ++ .../update_0/dep_2/kcl.mod.lock | 5 ++ .../test_run_with_mvs/update_0/dep_2/main.k | 1 + .../test_run_with_mvs/update_0/pkg/kcl.mod | 9 ++ .../update_0/pkg/kcl.mod.expect | 9 ++ .../update_0/pkg/kcl.mod.lock | 17 ++++ .../update_0/pkg/kcl.mod.lock.expect | 17 ++++ .../test_run_with_mvs/update_0/pkg/main.k | 1 + .../test_run_with_mvs/update_1/dep_0/kcl.mod | 7 ++ .../update_1/dep_0/kcl.mod.lock | 6 ++ .../test_run_with_mvs/update_1/dep_0/main.k | 1 + .../test_run_with_mvs/update_1/dep_1/kcl.mod | 7 ++ .../update_1/dep_1/kcl.mod.lock | 6 ++ .../test_run_with_mvs/update_1/dep_1/main.k | 1 + .../test_run_with_mvs/update_1/dep_2/kcl.mod | 7 ++ .../update_1/dep_2/kcl.mod.lock | 5 ++ .../test_run_with_mvs/update_1/dep_2/main.k | 1 + .../test_run_with_mvs/update_1/pkg/kcl.mod | 10 +++ .../test_run_with_mvs/update_1/pkg/kcl.mod.bk | 10 +++ .../update_1/pkg/kcl.mod.expect | 10 +++ .../update_1/pkg/kcl.mod.lock | 17 ++++ .../update_1/pkg/kcl.mod.lock.bk | 17 ++++ .../update_1/pkg/kcl.mod.lock.expect | 17 ++++ .../test_run_with_mvs/update_1/pkg/main.k | 1 + pkg/client/update_test.go | 2 +- pkg/downloader/downloader.go | 42 +++++++++- pkg/downloader/source.go | 4 + 43 files changed, 423 insertions(+), 35 deletions(-) create mode 100644 pkg/client/run_test.go create mode 100644 pkg/client/test_data/resolve_deps/my_kcl_compile/kcl.mod create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/dep_0/kcl.mod create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/dep_0/kcl.mod.lock create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/dep_0/main.k create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/dep_1/kcl.mod create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/dep_1/kcl.mod.lock create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/dep_1/main.k create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/dep_2/kcl.mod create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/dep_2/kcl.mod.lock create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/dep_2/main.k create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.expect create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.lock create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.lock.expect create mode 100644 pkg/client/test_data/test_run_with_mvs/update_0/pkg/main.k create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/dep_0/kcl.mod create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/dep_0/kcl.mod.lock create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/dep_0/main.k create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/dep_1/kcl.mod create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/dep_1/kcl.mod.lock create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/dep_1/main.k create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/dep_2/kcl.mod create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/dep_2/kcl.mod.lock create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/dep_2/main.k create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.bk create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.expect create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock.bk create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock.expect create mode 100644 pkg/client/test_data/test_run_with_mvs/update_1/pkg/main.k diff --git a/pkg/client/client.go b/pkg/client/client.go index 23779ad1..35748f11 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -325,6 +325,16 @@ func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) erro return err } + if update { + // Update the dependencies and select the version by mvs. + kclPkg.NoSumCheck = c.noSumCheck + _, err := c.Update( + WithUpdatedKclPkg(kclPkg), + ) + if err != nil { + return err + } + } } return nil } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index a60b5639..ab854a50 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -553,6 +553,7 @@ func TestVendorDeps(t *testing.T) { assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl1_0.0.1")), true) assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl2_0.0.1")), true) + kclPkg.SetVendorMode(true) maps, err := kpmcli.ResolveDepsIntoMap(&kclPkg) assert.Equal(t, err, nil) assert.Equal(t, len(maps), 2) @@ -592,6 +593,13 @@ func TestResolveDepsVendorMode(t *testing.T) { FullName: "kcl1_0.0.1", Version: "0.0.1", Sum: kcl1Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl1", + Tag: "0.0.1", + }, + }, } depKcl2 := pkg.Dependency{ @@ -599,6 +607,13 @@ func TestResolveDepsVendorMode(t *testing.T) { FullName: "kcl2_0.0.1", Version: "0.0.1", Sum: kcl2Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl2", + Tag: "0.0.1", + }, + }, } mppTest := orderedmap.NewOrderedMap[string, pkg.Dependency]() @@ -657,6 +672,13 @@ func TestCompileWithEntryFile(t *testing.T) { FullName: "kcl1_0.0.1", Version: "0.0.1", Sum: kcl1Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl1", + Tag: "0.0.1", + }, + }, } kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2")) depKcl2 := pkg.Dependency{ @@ -664,6 +686,13 @@ func TestCompileWithEntryFile(t *testing.T) { FullName: "kcl2_0.0.1", Version: "0.0.1", Sum: kcl2Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl2", + Tag: "0.0.1", + }, + }, } mppTest := orderedmap.NewOrderedMap[string, pkg.Dependency]() @@ -839,13 +868,13 @@ func TestTestResolveMetadataInJsonStrWithPackage(t *testing.T) { Deps: make(map[string]pkg.Dependency), } - localFullPath, err := utils.FindPackage(filepath.Join(globalPkgPath, "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8"), "helloworld") + localFullPath, err := utils.FindPackage(filepath.Join(globalPkgPath, "flask-demo-kcl-manifests_8308200"), "cc") assert.Equal(t, err, nil) - expectedDep.Deps["helloworld"] = pkg.Dependency{ - Name: "helloworld", - FullName: "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8", - Version: "ee03122b5f45b09eb48694422fc99a0772f6bba8", + expectedDep.Deps["cc"] = pkg.Dependency{ + Name: "cc", + FullName: "flask-demo-kcl-manifests_8308200", + Version: "8308200", LocalFullPath: localFullPath, } @@ -873,19 +902,19 @@ func TestTestResolveMetadataInJsonStrWithPackage(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, utils.DirExists(vendorDir), true) - assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8")), true) + assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "flask-demo-kcl-manifests_8308200")), true) - localFullPath, err = utils.FindPackage(filepath.Join(vendorDir, "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8"), "helloworld") + localFullPath, err = utils.FindPackage(filepath.Join(vendorDir, "flask-demo-kcl-manifests_8308200"), "cc") assert.Equal(t, err, nil) expectedDep = pkg.DependenciesUI{ Deps: make(map[string]pkg.Dependency), } - expectedDep.Deps["helloworld"] = pkg.Dependency{ - Name: "helloworld", - FullName: "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8", - Version: "ee03122b5f45b09eb48694422fc99a0772f6bba8", + expectedDep.Deps["cc"] = pkg.Dependency{ + Name: "cc", + FullName: "flask-demo-kcl-manifests_8308200", + Version: "8308200", LocalFullPath: localFullPath, } @@ -1139,7 +1168,7 @@ func TestMetadataOffline(t *testing.T) { beautifulContent, err := os.ReadFile(BeautifulKclMod) assert.Equal(t, err, nil) - kclPkg, err := pkg.LoadKclPkg(testDir) + kclPkg, err := kpmcli.LoadPkgFromPath(testDir) assert.Equal(t, err, nil) res, err := kpmcli.ResolveDepsMetadataInJsonStr(kclPkg, false) @@ -1204,8 +1233,9 @@ func TestRunWithGitPackage(t *testing.T) { opts.SetPkgPath(pkgPath) compileResult, err := kpmcli.CompileWithOpts(opts) + fmt.Printf("err: %v\n", err) assert.Equal(t, err, nil) - expectedCompileResult := `{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "web-app"}, "spec": {"containers": [{"image": "nginx", "name": "main-container", "ports": [{"containerPort": 80}]}]}}` + expectedCompileResult := `{"b": "Hello World!"}` assert.Equal(t, expectedCompileResult, compileResult.GetRawJsonResult()) assert.Equal(t, utils.DirExists(filepath.Join(pkgPath, "kcl.mod.lock")), true) diff --git a/pkg/client/resolver.go b/pkg/client/resolver.go index 87d1fb19..a8867003 100644 --- a/pkg/client/resolver.go +++ b/pkg/client/resolver.go @@ -1,6 +1,7 @@ package client import ( + "fmt" "path/filepath" "kcl-lang.io/kpm/pkg/downloader" @@ -96,6 +97,10 @@ func (dr *DepsResolver) Resolve(options ...ResolveOption) error { // For remote source, it will use the RemoteVisitor and enable the cache. // For local source, it will use the PkgVisitor. visitorSelectorFunc := func(source *downloader.Source) (Visitor, error) { + if source.IsNilSource() { + return nil, fmt.Errorf("the dependency source is nil") + } + if source.IsRemote() { PkgVisitor := NewRemoteVisitor(NewPkgVisitor(dr.kpmClient)) PkgVisitor.EnableCache = opts.EnableCache diff --git a/pkg/client/run_test.go b/pkg/client/run_test.go new file mode 100644 index 00000000..10d1ca92 --- /dev/null +++ b/pkg/client/run_test.go @@ -0,0 +1,82 @@ +package client + +import ( + "os" + "path/filepath" + "testing" + + "github.com/otiai10/copy" + "gotest.tools/v3/assert" + "kcl-lang.io/kpm/pkg/downloader" + "kcl-lang.io/kpm/pkg/utils" +) + +func TestRunWithMvs(t *testing.T) { + testDir := getTestDir("test_run_with_mvs") + kpmcli, err := NewKpmClient() + if err != nil { + t.Fatal(err) + } + + updates := []struct { + name string + before func() error + }{ + { + name: "update_0", + before: func() error { return nil }, + }, + { + name: "update_1", + before: func() error { + if err := copy.Copy(filepath.Join(testDir, "update_1", "pkg", "kcl.mod.bk"), filepath.Join(testDir, "update_1", "pkg", "kcl.mod")); err != nil { + return err + } + if err := copy.Copy(filepath.Join(testDir, "update_1", "pkg", "kcl.mod.lock.bk"), filepath.Join(testDir, "update_1", "pkg", "kcl.mod.lock")); err != nil { + return err + } + return nil + }, + }, + } + + for _, update := range updates { + if err := update.before(); err != nil { + t.Fatal(err) + } + + _, err = kpmcli.Run(WithRunSource( + &downloader.Source{ + Local: &downloader.Local{ + Path: filepath.Join(testDir, update.name, "pkg"), + }, + }, + )) + if err != nil { + t.Fatal(err) + } + + expectedMod, err := os.ReadFile(filepath.Join(testDir, update.name, "pkg", "kcl.mod.expect")) + if err != nil { + t.Fatal(err) + } + + expectedModLock, err := os.ReadFile(filepath.Join(testDir, update.name, "pkg", "kcl.mod.lock.expect")) + if err != nil { + t.Fatal(err) + } + + gotMod, err := os.ReadFile(filepath.Join(testDir, update.name, "pkg", "kcl.mod")) + if err != nil { + t.Fatal(err) + } + + gotModLock, err := os.ReadFile(filepath.Join(testDir, update.name, "pkg", "kcl.mod.lock")) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, utils.RmNewline(string(expectedMod)), utils.RmNewline(string(gotMod))) + assert.Equal(t, utils.RmNewline(string(expectedModLock)), utils.RmNewline(string(gotModLock))) + } +} diff --git a/pkg/client/test_data/resolve_deps/my_kcl_compile/kcl.mod b/pkg/client/test_data/resolve_deps/my_kcl_compile/kcl.mod new file mode 100644 index 00000000..0a9cbc2f --- /dev/null +++ b/pkg/client/test_data/resolve_deps/my_kcl_compile/kcl.mod @@ -0,0 +1,5 @@ +[package] + +[dependencies] +kcl1 = { oci = "oci://ghcr.io/kcl-lang/kcl1", tag = "0.0.1" } +kcl2 = { oci = "oci://ghcr.io/kcl-lang/kcl2", tag = "0.0.1" } diff --git a/pkg/client/test_data/resolve_deps/my_kcl_compile/kcl.mod.lock b/pkg/client/test_data/resolve_deps/my_kcl_compile/kcl.mod.lock index b6a76aec..959ed5aa 100644 --- a/pkg/client/test_data/resolve_deps/my_kcl_compile/kcl.mod.lock +++ b/pkg/client/test_data/resolve_deps/my_kcl_compile/kcl.mod.lock @@ -3,7 +3,13 @@ name = "kcl1" full_name = "kcl1_0.0.1" version = "0.0.1" + reg = "ghcr.io" + repo = "kcl-lang/kcl1" + oci_tag = "0.0.1" [dependencies.kcl2] name = "kcl2" full_name = "kcl2_0.0.1" version = "0.0.1" + reg = "ghcr.io" + repo = "kcl-lang/kcl2" + oci_tag = "0.0.1" diff --git a/pkg/client/test_data/resolve_metadata/with_package/kcl.mod b/pkg/client/test_data/resolve_metadata/with_package/kcl.mod index f6bc0ae4..c74af8a1 100644 --- a/pkg/client/test_data/resolve_metadata/with_package/kcl.mod +++ b/pkg/client/test_data/resolve_metadata/with_package/kcl.mod @@ -4,4 +4,4 @@ edition = "v0.8.0" version = "0.0.1" [dependencies] -helloworld = { git = "https://github.com/kcl-lang/modules.git", commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8", package = "helloworld" } +cc = { git = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", commit = "8308200", package = "cc" } diff --git a/pkg/client/test_data/resolve_metadata/with_package/kcl.mod.lock b/pkg/client/test_data/resolve_metadata/with_package/kcl.mod.lock index eb0223ed..32355765 100644 --- a/pkg/client/test_data/resolve_metadata/with_package/kcl.mod.lock +++ b/pkg/client/test_data/resolve_metadata/with_package/kcl.mod.lock @@ -1,8 +1,8 @@ [dependencies] - [dependencies.helloworld] - name = "helloworld" - full_name = "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8" - version = "0.1.2" - url = "https://github.com/kcl-lang/modules.git" - commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8" - package = "helloworld" + [dependencies.cc] + name = "cc" + full_name = "flask-demo-kcl-manifests_8308200" + version = "0.0.1" + url = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git" + commit = "8308200" + package = "cc" diff --git a/pkg/client/test_data/test_run_git_package/kcl.mod b/pkg/client/test_data/test_run_git_package/kcl.mod index 5e0b1cba..c74af8a1 100644 --- a/pkg/client/test_data/test_run_git_package/kcl.mod +++ b/pkg/client/test_data/test_run_git_package/kcl.mod @@ -4,4 +4,4 @@ edition = "v0.8.0" version = "0.0.1" [dependencies] -k8s = { git = "https://github.com/kcl-lang/modules.git", commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8", package = "k8s" } +cc = { git = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", commit = "8308200", package = "cc" } diff --git a/pkg/client/test_data/test_run_git_package/main.k b/pkg/client/test_data/test_run_git_package/main.k index d439ea05..926f6537 100644 --- a/pkg/client/test_data/test_run_git_package/main.k +++ b/pkg/client/test_data/test_run_git_package/main.k @@ -1,10 +1,3 @@ -import k8s.api.core.v1 as k8core +import cc -k8core.Pod { - metadata.name = "web-app" - spec.containers = [{ - name = "main-container" - image = "nginx" - ports = [{containerPort = 80}] - }] -} +b = cc.The_first_kcl_program \ No newline at end of file diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/dep_0/kcl.mod b/pkg/client/test_data/test_run_with_mvs/update_0/dep_0/kcl.mod new file mode 100644 index 00000000..14335b99 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/dep_0/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "dep_0" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +helloworld = "0.1.0" diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/dep_0/kcl.mod.lock b/pkg/client/test_data/test_run_with_mvs/update_0/dep_0/kcl.mod.lock new file mode 100644 index 00000000..1bc5d15c --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/dep_0/kcl.mod.lock @@ -0,0 +1,6 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.0" + version = "0.1.0" + sum = "aqrvSsd8zGHzRERbOzxYxARmK6QjvpQMYC1OqemdZvc=" diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/dep_0/main.k b/pkg/client/test_data/test_run_with_mvs/update_0/dep_0/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/dep_0/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/dep_1/kcl.mod b/pkg/client/test_data/test_run_with_mvs/update_0/dep_1/kcl.mod new file mode 100644 index 00000000..e4d487a9 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/dep_1/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "dep_1" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +helloworld = "0.1.1" diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/dep_1/kcl.mod.lock b/pkg/client/test_data/test_run_with_mvs/update_0/dep_1/kcl.mod.lock new file mode 100644 index 00000000..667cdff0 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/dep_1/kcl.mod.lock @@ -0,0 +1,6 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.1" + version = "0.1.1" + sum = "7OO4YK2QuRWPq9C7KTzcWcti5yUnueCjptT3OXiPVeQ=" diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/dep_1/main.k b/pkg/client/test_data/test_run_with_mvs/update_0/dep_1/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/dep_1/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/dep_2/kcl.mod b/pkg/client/test_data/test_run_with_mvs/update_0/dep_2/kcl.mod new file mode 100644 index 00000000..b098f6c7 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/dep_2/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "dep_2" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +helloworld = "0.1.0" diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/dep_2/kcl.mod.lock b/pkg/client/test_data/test_run_with_mvs/update_0/dep_2/kcl.mod.lock new file mode 100644 index 00000000..7b80eba3 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/dep_2/kcl.mod.lock @@ -0,0 +1,5 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.0" + version = "0.1.0" diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/dep_2/main.k b/pkg/client/test_data/test_run_with_mvs/update_0/dep_2/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/dep_2/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod b/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod new file mode 100644 index 00000000..c98ccd1b --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod @@ -0,0 +1,9 @@ +[package] +name = "pkg" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +dep_0 = { path = "../dep_0" } +dep_1 = { path = "../dep_1" } +dep_2 = { path = "../dep_2" } diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.expect b/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.expect new file mode 100644 index 00000000..c98ccd1b --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.expect @@ -0,0 +1,9 @@ +[package] +name = "pkg" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +dep_0 = { path = "../dep_0" } +dep_1 = { path = "../dep_1" } +dep_2 = { path = "../dep_2" } diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.lock b/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.lock new file mode 100644 index 00000000..e60448ad --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.lock @@ -0,0 +1,17 @@ +[dependencies] + [dependencies.dep_0] + name = "dep_0" + full_name = "dep_0_0.0.1" + version = "0.0.1" + [dependencies.dep_1] + name = "dep_1" + full_name = "dep_1_0.0.1" + version = "0.0.1" + [dependencies.dep_2] + name = "dep_2" + full_name = "dep_2_0.0.1" + version = "0.0.1" + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.1" + version = "0.1.1" diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.lock.expect b/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.lock.expect new file mode 100644 index 00000000..e60448ad --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/pkg/kcl.mod.lock.expect @@ -0,0 +1,17 @@ +[dependencies] + [dependencies.dep_0] + name = "dep_0" + full_name = "dep_0_0.0.1" + version = "0.0.1" + [dependencies.dep_1] + name = "dep_1" + full_name = "dep_1_0.0.1" + version = "0.0.1" + [dependencies.dep_2] + name = "dep_2" + full_name = "dep_2_0.0.1" + version = "0.0.1" + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.1" + version = "0.1.1" diff --git a/pkg/client/test_data/test_run_with_mvs/update_0/pkg/main.k b/pkg/client/test_data/test_run_with_mvs/update_0/pkg/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_0/pkg/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/dep_0/kcl.mod b/pkg/client/test_data/test_run_with_mvs/update_1/dep_0/kcl.mod new file mode 100644 index 00000000..14335b99 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/dep_0/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "dep_0" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +helloworld = "0.1.0" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/dep_0/kcl.mod.lock b/pkg/client/test_data/test_run_with_mvs/update_1/dep_0/kcl.mod.lock new file mode 100644 index 00000000..1bc5d15c --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/dep_0/kcl.mod.lock @@ -0,0 +1,6 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.0" + version = "0.1.0" + sum = "aqrvSsd8zGHzRERbOzxYxARmK6QjvpQMYC1OqemdZvc=" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/dep_0/main.k b/pkg/client/test_data/test_run_with_mvs/update_1/dep_0/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/dep_0/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/dep_1/kcl.mod b/pkg/client/test_data/test_run_with_mvs/update_1/dep_1/kcl.mod new file mode 100644 index 00000000..e4d487a9 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/dep_1/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "dep_1" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +helloworld = "0.1.1" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/dep_1/kcl.mod.lock b/pkg/client/test_data/test_run_with_mvs/update_1/dep_1/kcl.mod.lock new file mode 100644 index 00000000..667cdff0 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/dep_1/kcl.mod.lock @@ -0,0 +1,6 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.1" + version = "0.1.1" + sum = "7OO4YK2QuRWPq9C7KTzcWcti5yUnueCjptT3OXiPVeQ=" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/dep_1/main.k b/pkg/client/test_data/test_run_with_mvs/update_1/dep_1/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/dep_1/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/dep_2/kcl.mod b/pkg/client/test_data/test_run_with_mvs/update_1/dep_2/kcl.mod new file mode 100644 index 00000000..b098f6c7 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/dep_2/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "dep_2" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +helloworld = "0.1.0" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/dep_2/kcl.mod.lock b/pkg/client/test_data/test_run_with_mvs/update_1/dep_2/kcl.mod.lock new file mode 100644 index 00000000..7b80eba3 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/dep_2/kcl.mod.lock @@ -0,0 +1,5 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.0" + version = "0.1.0" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/dep_2/main.k b/pkg/client/test_data/test_run_with_mvs/update_1/dep_2/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/dep_2/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod new file mode 100644 index 00000000..e3d83935 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod @@ -0,0 +1,10 @@ +[package] +name = "pkg" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +dep_0 = { path = "../dep_0" } +dep_1 = { path = "../dep_1" } +dep_2 = { path = "../dep_2" } +helloworld = "0.1.1" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.bk b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.bk new file mode 100644 index 00000000..5e8f8790 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.bk @@ -0,0 +1,10 @@ +[package] +name = "pkg" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +dep_0 = { path = "../dep_0" } +dep_1 = { path = "../dep_1" } +dep_2 = { path = "../dep_2" } +helloworld = "0.1.0" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.expect b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.expect new file mode 100644 index 00000000..9d725b79 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.expect @@ -0,0 +1,10 @@ +[package] +name = "pkg" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +dep_0 = { path = "../dep_0" } +dep_1 = { path = "../dep_1" } +dep_2 = { path = "../dep_2" } +helloworld = "0.1.1" \ No newline at end of file diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock new file mode 100644 index 00000000..e60448ad --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock @@ -0,0 +1,17 @@ +[dependencies] + [dependencies.dep_0] + name = "dep_0" + full_name = "dep_0_0.0.1" + version = "0.0.1" + [dependencies.dep_1] + name = "dep_1" + full_name = "dep_1_0.0.1" + version = "0.0.1" + [dependencies.dep_2] + name = "dep_2" + full_name = "dep_2_0.0.1" + version = "0.0.1" + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.1" + version = "0.1.1" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock.bk b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock.bk new file mode 100644 index 00000000..04d67c17 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock.bk @@ -0,0 +1,17 @@ +[dependencies] + [dependencies.dep_0] + name = "dep_0" + full_name = "dep_0_0.0.1" + version = "0.0.1" + [dependencies.dep_1] + name = "dep_1" + full_name = "dep_1_0.0.1" + version = "0.0.1" + [dependencies.dep_2] + name = "dep_2" + full_name = "dep_2_0.0.1" + version = "0.0.1" + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.0" + version = "0.1.0" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock.expect b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock.expect new file mode 100644 index 00000000..e60448ad --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/kcl.mod.lock.expect @@ -0,0 +1,17 @@ +[dependencies] + [dependencies.dep_0] + name = "dep_0" + full_name = "dep_0_0.0.1" + version = "0.0.1" + [dependencies.dep_1] + name = "dep_1" + full_name = "dep_1_0.0.1" + version = "0.0.1" + [dependencies.dep_2] + name = "dep_2" + full_name = "dep_2_0.0.1" + version = "0.0.1" + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.1" + version = "0.1.1" diff --git a/pkg/client/test_data/test_run_with_mvs/update_1/pkg/main.k b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_run_with_mvs/update_1/pkg/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/update_test.go b/pkg/client/update_test.go index 1202428f..90f85fda 100644 --- a/pkg/client/update_test.go +++ b/pkg/client/update_test.go @@ -10,7 +10,7 @@ import ( "kcl-lang.io/kpm/pkg/utils" ) -func TestUpdate(t *testing.T) { +func TestUpdateWithMvs(t *testing.T) { testDir := getTestDir("test_update_with_mvs") kpmcli, err := NewKpmClient() if err != nil { diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go index 14580681..7ef42d59 100644 --- a/pkg/downloader/downloader.go +++ b/pkg/downloader/downloader.go @@ -129,6 +129,7 @@ func NewOciDownloader(platform string) *DepDownloader { func (d *DepDownloader) Download(opts DownloadOptions) error { var localPath string + originLocalPath := opts.LocalPath if opts.EnableCache { // TODO: After the new local storage structure is complete, // this section should be replaced with the new storage structure instead of the cache path according to the /. @@ -176,7 +177,20 @@ func (d *DepDownloader) Download(opts DownloadOptions) error { localPath = opts.LocalPath } - opts.LocalPath = localPath + // Create a temporary directory to download the package. + // The temporary directory will be cleaned after the download. + tmpDir, err := os.MkdirTemp("", "") + if err != nil { + return err + } + + if opts.Source.Git != nil { + tmpDir = filepath.Join(tmpDir, constants.GitScheme) + } + + defer os.RemoveAll(tmpDir) + + opts.LocalPath = tmpDir // Dispatch the download to the specific downloader by package source. if opts.Source.Oci != nil || opts.Source.Registry != nil { if opts.Source.Registry != nil { @@ -185,15 +199,37 @@ func (d *DepDownloader) Download(opts DownloadOptions) error { if d.OciDownloader == nil { d.OciDownloader = &OciDownloader{} } - return d.OciDownloader.Download(opts) + err := d.OciDownloader.Download(opts) + if err != nil { + return err + } } if opts.Source.Git != nil { if d.GitDownloader == nil { d.GitDownloader = &GitDownloader{} } - return d.GitDownloader.Download(opts) + err := d.GitDownloader.Download(opts) + if err != nil { + return err + } + } + + // Copy the tmpDir to the local path. + err = copy.Copy(tmpDir, localPath) + if err != nil { + return err + } + + // If the cache is enabled, the localpath is the cache path. + // Copy the cache to the origin local path user specified. + if opts.EnableCache { + err = copy.Copy(localPath, originLocalPath) + if err != nil { + return err + } } + return nil } diff --git a/pkg/downloader/source.go b/pkg/downloader/source.go index 7c173a74..332420ef 100644 --- a/pkg/downloader/source.go +++ b/pkg/downloader/source.go @@ -69,6 +69,10 @@ func (source *Source) IsLocalTgzPath() bool { return source.Local.IsLocalTgzPath() } +func (source *Source) IsNilSource() bool { + return source == nil || (source.Git == nil && source.Oci == nil && source.Local == nil && source.Registry == nil) +} + func (source *Source) IsRemote() bool { return source.Git != nil || source.Oci != nil || source.Registry != nil } From 7079b0b74243dfa4d8198e1d4f96c4ba8ebea6e6 Mon Sep 17 00:00:00 2001 From: zongz Date: Mon, 9 Sep 2024 16:32:21 +0800 Subject: [PATCH 2/6] feat: support mvs in vendor mode Signed-off-by: zongz --- pkg/api/kpm_pkg_test.go | 6 +- pkg/api/kpm_run.go | 1 + .../test_kpm_package/kcl_pkg/kcl.mod.lock | 4 - pkg/client/client.go | 54 +-- pkg/client/client_test.go | 29 +- pkg/client/resolver.go | 311 ++++++++++++++---- pkg/client/resolver_test.go | 9 +- pkg/client/run.go | 8 +- .../test_data/resolve_dep_with_kclmod/kcl.mod | 2 +- .../test_data/resolve_dep_with_kclmod/main.k | 11 +- .../test_metadata_offline/ugly.kcl.mod | 6 - .../test_pkg/expect.mod.lock | 2 +- .../test_pkg_win/expect.mod.lock | 2 +- .../test_run_options/no_args/run_0/kcl.mod | 2 - .../test_run_options/no_args/run_2/kcl.mod | 2 - .../test_run_options/no_args/run_4/kcl.mod | 2 +- .../test_run_options/no_args/run_8/kcl.mod | 1 - pkg/client/update.go | 139 +++++--- pkg/client/visitor.go | 280 +++++++++++++--- pkg/package/modfile.go | 3 - 20 files changed, 647 insertions(+), 227 deletions(-) diff --git a/pkg/api/kpm_pkg_test.go b/pkg/api/kpm_pkg_test.go index d5c41c8f..4278f937 100644 --- a/pkg/api/kpm_pkg_test.go +++ b/pkg/api/kpm_pkg_test.go @@ -30,9 +30,9 @@ func TestPackageApi(t *testing.T) { assert.Equal(t, dep.Name, "k8s") assert.Equal(t, dep.FullName, "k8s_1.27") assert.Equal(t, dep.Version, "1.27") - assert.Equal(t, dep.Source.Oci.Reg, "ghcr.io") - assert.Equal(t, dep.Source.Oci.Repo, "kcl-lang/k8s") - assert.Equal(t, dep.Source.Oci.Tag, "1.27") + assert.Equal(t, dep.Source.Registry.Oci.Reg, "ghcr.io") + assert.Equal(t, dep.Source.Registry.Oci.Repo, "kcl-lang/k8s") + assert.Equal(t, dep.Source.Registry.Oci.Tag, "1.27") assert.Equal(t, dep.GetLocalFullPath(""), filepath.Join(kcl_pkg_path, "k8s_1.27")) diff --git a/pkg/api/kpm_run.go b/pkg/api/kpm_run.go index 2f7dc953..1c12cf72 100644 --- a/pkg/api/kpm_run.go +++ b/pkg/api/kpm_run.go @@ -247,6 +247,7 @@ func run(kpmcli *client.KpmClient, opts *opt.CompileOptions) (*kcl.KCLResultList } kclPkg.SetVendorMode(opts.IsVendor()) + kclPkg.NoSumCheck = opts.NoSumCheck() globalPkgPath, err := env.GetAbsPkgPath() if err != nil { diff --git a/pkg/api/test_data/test_kpm_package/kcl_pkg/kcl.mod.lock b/pkg/api/test_data/test_kpm_package/kcl_pkg/kcl.mod.lock index 7d73c530..feb5c660 100644 --- a/pkg/api/test_data/test_kpm_package/kcl_pkg/kcl.mod.lock +++ b/pkg/api/test_data/test_kpm_package/kcl_pkg/kcl.mod.lock @@ -3,7 +3,3 @@ name = "k8s" full_name = "k8s_1.27" version = "1.27" - sum = "xnYM1FWHAy3m+KcQMQb2rjZouTxumqYt6FGZpu2T4yM=" - reg = "ghcr.io" - repo = "kcl-lang/k8s" - oci_tag = "1.27" diff --git a/pkg/client/client.go b/pkg/client/client.go index 35748f11..c767d0a6 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -312,29 +312,14 @@ func (c *KpmClient) getDepStorePath(search_path string, d *pkg.Dependency, isVen // Since redownloads are not triggered if local dependencies exists, // indirect dependencies are also synchronized to the lock file by `lockDeps`. func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) error { - if kclPkg.IsVendorMode() { - // In the vendor mode, the search path is the vendor subdirectory of the current package. - err := c.VendorDeps(kclPkg) - if err != nil { - return err - } - } else { - // In the non-vendor mode, the search path is the KCL_PKG_PATH. - err := c.resolvePkgDeps(kclPkg, &kclPkg.Dependencies, update) - if err != nil { - return err - } - - if update { - // Update the dependencies and select the version by mvs. - kclPkg.NoSumCheck = c.noSumCheck - _, err := c.Update( - WithUpdatedKclPkg(kclPkg), - ) - if err != nil { - return err - } - } + _, err := c.Update( + WithUpdatedKclPkg(kclPkg), + WithUpdateEnableVendor(kclPkg.IsVendorMode()), + WithUpdateVendorPath(kclPkg.LocalVendorPath()), + WithUpdateOffline(!update), + ) + if err != nil { + return err } return nil } @@ -456,6 +441,17 @@ func (c *KpmClient) resolvePkgDeps(kclPkg *pkg.KclPkg, lockDeps *pkg.Dependencie } } + if update { + // Update the dependencies and select the version by mvs. + kclPkg.NoSumCheck = c.noSumCheck + _, err := c.Update( + WithUpdatedKclPkg(kclPkg), + ) + if err != nil { + return err + } + } + return nil } @@ -483,10 +479,6 @@ func (c *KpmClient) UpdateDeps(kclPkg *pkg.KclPkg) error { return err } - _, err = c.Update( - WithUpdatedKclPkg(kclPkg), - ) - return err } @@ -548,6 +540,7 @@ func (c *KpmClient) CompileWithOpts(opts *opt.CompileOptions) (*kcl.KCLResultLis } kclPkg.SetVendorMode(opts.IsVendor()) + kclPkg.NoSumCheck = opts.NoSumCheck() globalPkgPath, err := env.GetAbsPkgPath() if err != nil { @@ -931,6 +924,13 @@ func (c *KpmClient) vendorDeps(kclPkg *pkg.KclPkg, vendorPath string) error { // If the package already exists in the 'vendor', do nothing. if utils.DirExists(vendorFullPath) { + if d.GetPackage() != "" { + tempVendorFullPath, err := utils.FindPackage(vendorFullPath, d.GetPackage()) + if err != nil { + return err + } + vendorFullPath = tempVendorFullPath + } d.LocalFullPath = vendorFullPath lockDeps[i] = d continue diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index ab854a50..1f1f4811 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -510,6 +510,13 @@ func TestVendorDeps(t *testing.T) { FullName: "kcl1", Version: "0.0.1", Sum: kcl1Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl1", + Tag: "0.0.1", + }, + }, } depKcl2 := pkg.Dependency{ @@ -517,6 +524,13 @@ func TestVendorDeps(t *testing.T) { FullName: "kcl2", Version: "0.0.1", Sum: kcl2Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl2", + Tag: "0.0.1", + }, + }, } mppTest := orderedmap.NewOrderedMap[string, pkg.Dependency]() @@ -572,8 +586,8 @@ func TestResolveDepsWithOnlyKclMod(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, len(depsMap), 1) assert.Equal(t, utils.DirExists(filepath.Join(testDir, "kcl.mod.lock")), true) - assert.Equal(t, depsMap["k8s"], filepath.Join(kpmcli.homePath, "k8s_1.17")) - assert.Equal(t, utils.DirExists(filepath.Join(kpmcli.homePath, "k8s_1.17")), true) + assert.Equal(t, depsMap["helloworld"], filepath.Join(kpmcli.homePath, "helloworld_0.1.2")) + assert.Equal(t, utils.DirExists(filepath.Join(kpmcli.homePath, "helloworld_0.1.2")), true) defer func() { err := os.Remove(filepath.Join(testDir, "kcl.mod.lock")) assert.Equal(t, err, nil) @@ -839,7 +853,7 @@ func TestResolveMetadataInJsonStr(t *testing.T) { assert.Equal(t, utils.DirExists(vendorDir), false) assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "flask-demo-kcl-manifests_ade147b")), false) assert.Equal(t, err, nil) - expectedStr := "{\"packages\":{\"flask_demo_kcl_manifests\":{\"name\":\"flask_demo_kcl_manifests\",\"manifest_path\":\"\"}}}" + expectedStr := "{\"packages\":{\"flask_demo_kcl_manifests\":{\"name\":\"flask_demo_kcl_manifests\",\"manifest_path\":\"not_exist/flask-demo-kcl-manifests_ade147b\"}}}" assert.Equal(t, res, expectedStr) defer func() { if r := os.RemoveAll(filepath.Join("not_exist", "flask-demo-kcl-manifests_ade147b")); r != nil { @@ -1324,6 +1338,7 @@ func TestUpdateWithNoSumCheck(t *testing.T) { kclPkg, err := kpmcli.LoadPkgFromPath(pkgPath) assert.Equal(t, err, nil) + kclPkg.NoSumCheck = true err = kpmcli.UpdateDeps(kclPkg) assert.Equal(t, err, nil) assert.Equal(t, utils.DirExists(filepath.Join(pkgPath, "kcl.mod.lock")), false) @@ -2290,14 +2305,18 @@ func TestVirtualPackageVisiter(t *testing.T) { assert.Equal(t, err, nil) v := NewVisitor(*pkgSource, kpmcli) - err = v.Visit(pkgSource, func(p *pkg.KclPkg) error { + visitFunc := func(p *pkg.KclPkg) error { assert.Contains(t, p.GetPkgName(), "vPkg_") _, err = os.Stat(filepath.Join(pkgPath, "kcl.mod")) assert.Equal(t, os.IsNotExist(err), true) _, err = os.Stat(filepath.Join(pkgPath, "kcl.mod.lock")) assert.Equal(t, os.IsNotExist(err), true) return nil - }) + } + err = v.Visit( + WithVisitSource(pkgSource), + WithVisitFunc(visitFunc), + ) assert.Equal(t, err, nil) _, err = os.Stat(filepath.Join(pkgPath, "kcl.mod")) assert.Equal(t, os.IsNotExist(err), true) diff --git a/pkg/client/resolver.go b/pkg/client/resolver.go index a8867003..5a9e8c94 100644 --- a/pkg/client/resolver.go +++ b/pkg/client/resolver.go @@ -2,10 +2,15 @@ package client import ( "fmt" + "os" "path/filepath" + "strings" + "github.com/otiai10/copy" + "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/downloader" pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/utils" ) // ResolveOption is the option for resolving dependencies. @@ -16,13 +21,43 @@ type ResolveOption func(*ResolveOptions) error type resolveFunc func(dep *pkg.Dependency, parentPkg *pkg.KclPkg) error type ResolveOptions struct { - // Source is the source of the package to be pulled. - // Including git, oci, local. - Source *downloader.Source + + // Source is the source of the package to be resolved. + kpkg *pkg.KclPkg // EnableCache is the flag to enable the cache during the resolving the remote package. EnableCache bool + // EnableVendor is the flag to enable the vendor. + EnableVendor bool // CachePath is the path of the cache. CachePath string + // vendorPath is the path of the vendor. + VendorPath string + // Offline is the flag to enable the offline mode. + Offline bool +} + +// WithOffline sets the flag to enable the offline mode. +func WithOffline(offline bool) ResolveOption { + return func(opts *ResolveOptions) error { + opts.Offline = offline + return nil + } +} + +// WithEnableVendor sets the flag to enable the vendor. +func WithEnableVendor(enableVendor bool) ResolveOption { + return func(opts *ResolveOptions) error { + opts.EnableVendor = enableVendor + return nil + } +} + +// WithVendorPath sets the path of the vendor. +func WithVendorPath(vendorPath string) ResolveOption { + return func(opts *ResolveOptions) error { + opts.VendorPath = vendorPath + return nil + } } // WithEnableCache sets the flag to enable the cache during the resolving the remote package. @@ -41,22 +76,10 @@ func WithCachePath(cachePath string) ResolveOption { } } -// WithResolveSource sets the source of the package to be resolved. -func WithResolveSource(source *downloader.Source) ResolveOption { - return func(opts *ResolveOptions) error { - opts.Source = source - return nil - } -} - -// WithResolveSourceUrl sets the source of the package to be resolved by the source url. -func WithResolveSourceUrl(sourceUrl string) ResolveOption { +// WithKclPkg sets the kcl package to be resolved. +func WithResolveKclPkg(kpkg *pkg.KclPkg) ResolveOption { return func(opts *ResolveOptions) error { - source, err := downloader.NewSourceFromStr(sourceUrl) - if err != nil { - return err - } - opts.Source = source + opts.kpkg = kpkg return nil } } @@ -93,6 +116,131 @@ func (dr *DepsResolver) Resolve(options ...ResolveOption) error { } } + var accessFunc accessFunc + + // A custom function to download the package consider the offline mode. + downloadPkgTo := func(source *downloader.Source, localPath string) error { + credCli, err := dr.kpmClient.GetCredsClient() + if err != nil { + return err + } + + var pkgPath string + + if !opts.Offline { + tmpDir, err := os.MkdirTemp("", "") + if err != nil { + return err + } + + if source.Git != nil { + tmpDir = filepath.Join(tmpDir, constants.GitScheme) + } + defer os.RemoveAll(tmpDir) + err = dr.kpmClient.DepDownloader.Download(*downloader.NewDownloadOptions( + downloader.WithLocalPath(tmpDir), + downloader.WithSource(*source), + downloader.WithLogWriter(dr.kpmClient.GetLogWriter()), + downloader.WithSettings(*dr.kpmClient.GetSettings()), + downloader.WithCredsClient(credCli), + downloader.WithCachePath(opts.CachePath), + downloader.WithEnableCache(opts.EnableCache), + downloader.WithInsecureSkipTLSverify(dr.kpmClient.insecureSkipTLSverify), + )) + + if err != nil { + return err + } + pkgPath = tmpDir + if source.Git != nil && len(source.Git.Package) > 0 { + pkgPath, err = utils.FindPackage(tmpDir, source.Git.Package) + if err != nil { + return err + } + } + } else { + pkgPath = filepath.Join(opts.CachePath, cacheFileNameFromSource(source)) + if !utils.DirExists(pkgPath) { + return fmt.Errorf("package not found in the %s", pkgPath) + } else { + if source.Git != nil && len(source.Git.Package) > 0 { + pkgPath, err = utils.FindPackage(pkgPath, source.Git.Package) + if err != nil { + return err + } + } + } + } + + if !utils.DirExists(localPath) { + err = copy.Copy(pkgPath, localPath) + if err != nil { + return err + } + } + return nil + } + + // A custom function to access the package consider the vendor path. + accessFunc = func(source *downloader.Source) (*pkg.KclPkg, error) { + if opts.EnableVendor { + vendorFullPath := filepath.Join(opts.VendorPath, cacheFileNameFromSource(source)) + if utils.DirExists(vendorFullPath) { + if source.GetPackage() != "" { + tempVendorFullPath, err := utils.FindPackage(vendorFullPath, source.GetPackage()) + if err != nil { + return nil, err + } + vendorFullPath = tempVendorFullPath + } + return dr.kpmClient.LoadPkgFromPath(vendorFullPath) + } else { + downloadPkgTo(source, vendorFullPath) + if source.GetPackage() != "" { + tempVendorFullPath, err := utils.FindPackage(vendorFullPath, source.GetPackage()) + if err != nil { + return nil, err + } + vendorFullPath = tempVendorFullPath + } + kclPkg, err := dr.kpmClient.LoadPkgFromPath(vendorFullPath) + if err != nil { + return nil, err + } + return kclPkg, nil + } + } else { + cacheFullPath := filepath.Join(opts.CachePath, cacheFileNameFromSource(source)) + if utils.DirExists(cacheFullPath) && utils.DirExists(filepath.Join(cacheFullPath, constants.KCL_MOD)) { + if source.GetPackage() != "" { + tempCacheFullPath, err := utils.FindPackage(cacheFullPath, source.GetPackage()) + if err != nil { + return nil, err + } + cacheFullPath = tempCacheFullPath + } + return dr.kpmClient.LoadPkgFromPath(cacheFullPath) + } else { + err := os.MkdirAll(cacheFullPath, 0755) + if err != nil { + return nil, err + } + err = downloadPkgTo(source, cacheFullPath) + if err != nil { + return nil, err + } + if source.GetPackage() != "" { + tempCacheFullPath, err := utils.FindPackage(cacheFullPath, source.GetPackage()) + if err != nil { + return nil, err + } + cacheFullPath = tempCacheFullPath + } + return dr.kpmClient.LoadPkgFromPath(cacheFullPath) + } + } + } + // visitorSelectorFunc selects the visitor for the source. // For remote source, it will use the RemoteVisitor and enable the cache. // For local source, it will use the PkgVisitor. @@ -111,74 +259,111 @@ func (dr *DepsResolver) Resolve(options ...ResolveOption) error { } return PkgVisitor, nil } else { + accessFunc = nil return NewVisitor(*source, dr.kpmClient), nil } } - // visitFunc is the function for visiting the package. - // It will traverse the dependency graph and visit each dependency by source. - visitFunc := func(kclPkg *pkg.KclPkg) error { - // Traverse the all dependencies of the package. - for _, depKey := range kclPkg.ModFile.Deps.Keys() { - dep, ok := kclPkg.ModFile.Deps.Get(depKey) - if !ok { - break - } + kpkg := opts.kpkg + if kpkg == nil { + return fmt.Errorf("kcl package is nil") + } - // Get the dependency source. - var depSource downloader.Source - // If the dependency source is a local path and the path is not absolute, transform the path to absolute path. - if dep.Source.IsLocalPath() && !filepath.IsAbs(dep.Source.Path) { - depSource = downloader.Source{ - Local: &downloader.Local{ - Path: filepath.Join(kclPkg.HomePath, dep.Source.Path), - }, - } - } else { - depSource = dep.Source - } + modDeps := kpkg.ModFile.Dependencies.Deps + if modDeps == nil { + return fmt.Errorf("kcl.mod dependencies is nil") + } - // Get the visitor for the dependency source. - visitor, err := visitorSelectorFunc(&depSource) - if err != nil { - return err - } + // Iterate all the dependencies of the package in kcl.mod and resolve each dependency. + for _, depName := range modDeps.Keys() { + dep, ok := modDeps.Get(depName) + if !ok { + return fmt.Errorf("failed to get dependency %s", depName) + } - // Visit this dependency and current package as the parent package. - err = visitor.Visit(&depSource, - func(childPkg *pkg.KclPkg) error { - for _, resolveFunc := range dr.resolveFuncs { - err := resolveFunc(&dep, kclPkg) - if err != nil { - return err - } - } - return nil + // Check if the dependency is a local path and it is not an absolute path. + // If it is not an absolute path, transform the path to an absolute path. + var depSource *downloader.Source + if dep.Source.IsLocalPath() && !filepath.IsAbs(dep.Source.Local.Path) { + depSource = &downloader.Source{ + Local: &downloader.Local{ + Path: filepath.Join(kpkg.HomePath, dep.Source.Local.Path), }, - ) + } + } else { + depSource = &dep.Source + } - if err != nil { - return err + visitor, err := visitorSelectorFunc(depSource) + if err != nil { + return err + } + + // visitFunc is the function for visiting the package. + // It will traverse the dependency graph and visit each dependency by source. + visitFunc := func(kclPkg *pkg.KclPkg) error { + dep.FromKclPkg(kclPkg) + for _, resolveFunc := range dr.resolveFuncs { + err := resolveFunc(&dep, kpkg) + if err != nil { + return err + } } // Recursively resolve the dependencies of the dependency. err = dr.Resolve( - WithResolveSource(&depSource), + WithResolveKclPkg(kclPkg), WithEnableCache(opts.EnableCache), WithCachePath(opts.CachePath), ) if err != nil { return err } + // } + + return nil } - return nil + err = visitor.Visit( + WithVisitSource(depSource), + WithVisitFunc(visitFunc), + WithAccessFunc(accessFunc), + ) + + if err != nil { + return err + } } - visitor, err := visitorSelectorFunc(opts.Source) - if err != nil { - return err + return nil +} + +// TODO: After the new local storage structure is complete, +// +// this section should be replaced with the new storage structure instead of the cache path according to the /. +// +// https://github.com/kcl-lang/kpm/issues/384 +func cacheFileNameFromSource(source *downloader.Source) string { + var pkgFullName string + if source.Registry != nil && len(source.Registry.Version) != 0 { + pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(source.Registry.Oci.Repo), source.Registry.Version) + } + if source.Oci != nil && len(source.Oci.Tag) != 0 { + pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(source.Oci.Repo), source.Oci.Tag) + } + + if source.Git != nil && len(source.Git.Tag) != 0 { + gitUrl := strings.TrimSuffix(source.Git.Url, filepath.Ext(source.Git.Url)) + pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(gitUrl), source.Git.Tag) + } + if source.Git != nil && len(source.Git.Branch) != 0 { + gitUrl := strings.TrimSuffix(source.Git.Url, filepath.Ext(source.Git.Url)) + pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(gitUrl), source.Git.Branch) + } + if source.Git != nil && len(source.Git.Commit) != 0 { + gitUrl := strings.TrimSuffix(source.Git.Url, filepath.Ext(source.Git.Url)) + pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(gitUrl), source.Git.Commit) } - return visitor.Visit(opts.Source, visitFunc) + return pkgFullName } diff --git a/pkg/client/resolver_test.go b/pkg/client/resolver_test.go index fa7773e5..f2cc0880 100644 --- a/pkg/client/resolver_test.go +++ b/pkg/client/resolver_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "kcl-lang.io/kpm/pkg/downloader" pkg "kcl-lang.io/kpm/pkg/package" ) @@ -20,8 +19,7 @@ func TestResolver(t *testing.T) { resolve_path := getTestDir("test_resolve_graph") pkgPath := filepath.Join(resolve_path, "pkg") - - pkgSource, err := downloader.NewSourceFromStr(pkgPath) + kpkg, err := kpmcli.LoadPkgFromPath(pkgPath) if err != nil { t.Fatal(err) } @@ -38,7 +36,8 @@ func TestResolver(t *testing.T) { err = resolver.Resolve( WithEnableCache(true), - WithResolveSource(pkgSource), + WithCachePath(kpmcli.homePath), + WithResolveKclPkg(kpkg), ) if err != nil { @@ -54,5 +53,5 @@ func TestResolver(t *testing.T) { sort.Strings(res) assert.Equal(t, len(res), 3) assert.Equal(t, res, expected) - assert.Equal(t, buf.String(), "") + assert.Contains(t, buf.String(), "") } diff --git a/pkg/client/run.go b/pkg/client/run.go index b7765331..7323ef1e 100644 --- a/pkg/client/run.go +++ b/pkg/client/run.go @@ -571,7 +571,7 @@ func (c *KpmClient) Run(options ...RunOption) (*kcl.KCLResultList, error) { // Visit the root package source. var res *kcl.KCLResultList - err = NewVisitor(*pkgSource, c).Visit(pkgSource, func(kclPkg *pkg.KclPkg) error { + visitFunc := func(kclPkg *pkg.KclPkg) error { // Apply the compile options from cli, kcl.yaml or kcl.mod err = opts.applyCompileOptions(*pkgSource, kclPkg, opts.WorkDir) if err != nil { @@ -602,7 +602,11 @@ func (c *KpmClient) Run(options ...RunOption) (*kcl.KCLResultList, error) { } return nil - }) + } + err = NewVisitor(*pkgSource, c).Visit( + WithVisitSource(pkgSource), + WithVisitFunc(visitFunc), + ) if err != nil { return nil, err diff --git a/pkg/client/test_data/resolve_dep_with_kclmod/kcl.mod b/pkg/client/test_data/resolve_dep_with_kclmod/kcl.mod index 1724ca41..6374192c 100644 --- a/pkg/client/test_data/resolve_dep_with_kclmod/kcl.mod +++ b/pkg/client/test_data/resolve_dep_with_kclmod/kcl.mod @@ -4,4 +4,4 @@ edition = "0.0.1" version = "0.0.1" [dependencies] -k8s = "1.17" +helloworld = "0.1.2" diff --git a/pkg/client/test_data/resolve_dep_with_kclmod/main.k b/pkg/client/test_data/resolve_dep_with_kclmod/main.k index 7f748048..8b03a3af 100644 --- a/pkg/client/test_data/resolve_dep_with_kclmod/main.k +++ b/pkg/client/test_data/resolve_dep_with_kclmod/main.k @@ -1,10 +1,3 @@ -import k8s.api.core.v1 as k8core +import helloworld -k8core.Pod { - metadata.name = "web-app" - spec.containers = [{ - name = "main-container" - image = "nginx" - ports = [{containerPort = 80}] - }] -} \ No newline at end of file +a = helloworld.The_first_kcl_program \ No newline at end of file diff --git a/pkg/client/test_data/test_metadata_offline/ugly.kcl.mod b/pkg/client/test_data/test_metadata_offline/ugly.kcl.mod index ea2fabea..718782c3 100644 --- a/pkg/client/test_data/test_metadata_offline/ugly.kcl.mod +++ b/pkg/client/test_data/test_metadata_offline/ugly.kcl.mod @@ -1,10 +1,4 @@ [package] - - - - - name = "test_metadata_offline" edition = "0.0.1" version = "0.0.1" - diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod.lock b/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod.lock index 32355765..2d3841c0 100644 --- a/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod.lock +++ b/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod.lock @@ -1,7 +1,7 @@ [dependencies] [dependencies.cc] name = "cc" - full_name = "flask-demo-kcl-manifests_8308200" + full_name = "cc_0.0.1" version = "0.0.1" url = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git" commit = "8308200" diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod.lock b/pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod.lock index 32355765..2d3841c0 100644 --- a/pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod.lock +++ b/pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod.lock @@ -1,7 +1,7 @@ [dependencies] [dependencies.cc] name = "cc" - full_name = "flask-demo-kcl-manifests_8308200" + full_name = "cc_0.0.1" version = "0.0.1" url = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git" commit = "8308200" diff --git a/pkg/client/test_data/test_run_options/no_args/run_0/kcl.mod b/pkg/client/test_data/test_run_options/no_args/run_0/kcl.mod index f8b4cf5b..fb3440b1 100644 --- a/pkg/client/test_data/test_run_options/no_args/run_0/kcl.mod +++ b/pkg/client/test_data/test_run_options/no_args/run_0/kcl.mod @@ -2,5 +2,3 @@ name = "run_0" edition = "v0.10.0" version = "0.0.1" - - diff --git a/pkg/client/test_data/test_run_options/no_args/run_2/kcl.mod b/pkg/client/test_data/test_run_options/no_args/run_2/kcl.mod index 78195993..a2834c6a 100644 --- a/pkg/client/test_data/test_run_options/no_args/run_2/kcl.mod +++ b/pkg/client/test_data/test_run_options/no_args/run_2/kcl.mod @@ -2,5 +2,3 @@ name = "run_2" edition = "v0.10.0" version = "0.0.1" - - diff --git a/pkg/client/test_data/test_run_options/no_args/run_4/kcl.mod b/pkg/client/test_data/test_run_options/no_args/run_4/kcl.mod index d7d0f4c3..16a854d4 100644 --- a/pkg/client/test_data/test_run_options/no_args/run_4/kcl.mod +++ b/pkg/client/test_data/test_run_options/no_args/run_4/kcl.mod @@ -4,4 +4,4 @@ edition = "v0.10.0" version = "0.0.1" [dependencies] -run_5 = "0.0.1" \ No newline at end of file +run_5 = "0.0.1" diff --git a/pkg/client/test_data/test_run_options/no_args/run_8/kcl.mod b/pkg/client/test_data/test_run_options/no_args/run_8/kcl.mod index 40f9e490..3d9fd1c2 100644 --- a/pkg/client/test_data/test_run_options/no_args/run_8/kcl.mod +++ b/pkg/client/test_data/test_run_options/no_args/run_8/kcl.mod @@ -2,4 +2,3 @@ name = "run_8" edition = "v0.10.0" version = "0.0.1" - diff --git a/pkg/client/update.go b/pkg/client/update.go index 9414e992..ee423cea 100644 --- a/pkg/client/update.go +++ b/pkg/client/update.go @@ -4,19 +4,47 @@ import ( "fmt" "path/filepath" - "kcl-lang.io/kpm/pkg/downloader" + "github.com/elliotchance/orderedmap/v2" pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/utils" ) // UpdateOptions is the option for updating a package. // Updating a package means iterating all the dependencies of the package // and updating the dependencies and selecting the version of the dependencies by MVS. type UpdateOptions struct { - kpkg *pkg.KclPkg + kpkg *pkg.KclPkg + enableVendor bool + vendorPath string + offline bool } type UpdateOption func(*UpdateOptions) error +// WithOffline sets the flag to enable the offline mode. +func WithUpdateOffline(offline bool) UpdateOption { + return func(opts *UpdateOptions) error { + opts.offline = offline + return nil + } +} + +// WithEnableVendor sets the flag to enable the vendor. +func WithUpdateEnableVendor(enableVendor bool) UpdateOption { + return func(opts *UpdateOptions) error { + opts.enableVendor = enableVendor + return nil + } +} + +// WithVendorPath sets the path of the vendor. +func WithUpdateVendorPath(vendorPath string) UpdateOption { + return func(opts *UpdateOptions) error { + opts.vendorPath = vendorPath + return nil + } +} + // WithUpdatedKclPkg sets the kcl package to be updated. func WithUpdatedKclPkg(kpkg *pkg.KclPkg) UpdateOption { return func(opts *UpdateOptions) error { @@ -42,13 +70,40 @@ func (c *KpmClient) Update(options ...UpdateOption) (*pkg.KclPkg, error) { if modDeps == nil { return nil, fmt.Errorf("kcl.mod dependencies is nil") } - lockDeps := kpkg.Dependencies.Deps - if lockDeps == nil { - return nil, fmt.Errorf("kcl.mod.lock dependencies is nil") - } + lockDeps := orderedmap.NewOrderedMap[string, pkg.Dependency]() // Create a new dependency resolver resolver := NewDepsResolver(c) + // TODO: Update the local path of the dependency. + // After the new local storage structure is complete, + // this section should be replaced with the new storage structure instead of the cache path according to the /. + // https://github.com/kcl-lang/kpm/issues/384 + resolvePathFunc := func(dep *pkg.Dependency, kpkg *pkg.KclPkg) error { + if opts.enableVendor { + dep.LocalFullPath = filepath.Join(opts.vendorPath, dep.GenPathSuffix()) + } else { + dep.LocalFullPath = filepath.Join(c.homePath, dep.GenPathSuffix()) + } + + var err error + if dep.Source.Git != nil && len(dep.Source.Git.Package) > 0 { + dep.LocalFullPath, err = utils.FindPackage(dep.LocalFullPath, dep.Source.Git.Package) + if err != nil { + return err + } + } + + if dep.Source.IsLocalPath() { + if !filepath.IsAbs(dep.Source.Local.Path) { + dep.LocalFullPath = filepath.Join(kpkg.HomePath, dep.Source.Local.Path) + } else { + dep.LocalFullPath = dep.Source.Local.Path + } + } + + return nil + } + // ResolveFunc is the function for resolving each dependency when traversing the dependency graph. resolverFunc := func(dep *pkg.Dependency, parentPkg *pkg.KclPkg) error { // Check if the dependency exists in the mod file. @@ -56,7 +111,17 @@ func (c *KpmClient) Update(options ...UpdateOption) (*pkg.KclPkg, error) { // if the dependency exists in the mod file, // check the version and select the greater one. if less, err := existDep.VersionLessThan(dep); less && err == nil { - kpkg.ModFile.Dependencies.Deps.Set(dep.Name, *dep) + err = resolvePathFunc(dep, parentPkg) + if err != nil { + return err + } + modDeps.Set(dep.Name, *dep) + } else { + err = resolvePathFunc(&existDep, parentPkg) + if err != nil { + return err + } + modDeps.Set(dep.Name, existDep) } // if the dependency does not exist in the mod file, // the dependency is a indirect dependency. @@ -67,48 +132,48 @@ func (c *KpmClient) Update(options ...UpdateOption) (*pkg.KclPkg, error) { // If the dependency exists in the lock file, // check the version and select the greater one. if less, err := existDep.VersionLessThan(dep); less && err == nil { - kpkg.Dependencies.Deps.Set(dep.Name, *dep) + err = resolvePathFunc(dep, parentPkg) + if err != nil { + return err + } + lockDeps.Set(dep.Name, *dep) + } else { + err = resolvePathFunc(&existDep, parentPkg) + if err != nil { + return err + } + lockDeps.Set(dep.Name, existDep) } } else { // if the dependency does not exist in the lock file, // the dependency is a new dependency and will be added to the lock file. - kpkg.Dependencies.Deps.Set(dep.Name, *dep) + err := resolvePathFunc(dep, parentPkg) + if err != nil { + return err + } + lockDeps.Set(dep.Name, *dep) } return nil } - resolver.AddResolveFunc(resolverFunc) - - // Iterate all the dependencies of the package in kcl.mod and resolve each dependency. - for _, depName := range modDeps.Keys() { - dep, ok := modDeps.Get(depName) - if !ok { - return nil, fmt.Errorf("failed to get dependency %s", depName) - } - // Check if the dependency is a local path and it is not an absolute path. - // If it is not an absolute path, transform the path to an absolute path. - var depSource *downloader.Source - if dep.Source.IsLocalPath() && !filepath.IsAbs(dep.Source.Local.Path) { - depSource = &downloader.Source{ - Local: &downloader.Local{ - Path: filepath.Join(kpkg.HomePath, dep.Source.Local.Path), - }, - } - } else { - depSource = &dep.Source - } + resolver.AddResolveFunc(resolverFunc) - err := resolver.Resolve( - WithEnableCache(true), - WithResolveSource(depSource), - ) - if err != nil { - return nil, err - } + err := resolver.Resolve( + WithEnableCache(true), + WithCachePath(c.homePath), + WithResolveKclPkg(kpkg), + WithEnableVendor(opts.enableVendor), + WithVendorPath(opts.vendorPath), + ) + if err != nil { + return nil, err } - err := kpkg.UpdateModAndLockFile() + kpkg.Dependencies.Deps = lockDeps + kpkg.ModFile.Dependencies.Deps = modDeps + + err = kpkg.UpdateModAndLockFile() if err != nil { return nil, err } diff --git a/pkg/client/visitor.go b/pkg/client/visitor.go index 0d151ded..a6cf60b0 100644 --- a/pkg/client/visitor.go +++ b/pkg/client/visitor.go @@ -13,11 +13,51 @@ import ( "kcl-lang.io/kpm/pkg/utils" ) +type VisitOptions struct { + Source *downloader.Source + // The funcs to visit the package. + visitFuncs []visitFunc + // The func to download/access the package. + // For remote package, it will download the package. + // For local package, it will find the package. + accessFunc *accessFunc +} + +type VisitOption func(*VisitOptions) error + +// The func help to download or find the package from the source. +type accessFunc func(source *downloader.Source) (*pkg.KclPkg, error) + +// The func help to visit the package after access the package. type visitFunc func(pkg *pkg.KclPkg) error +// WithVisitFunc sets the visitFunc for visiting the package. +func WithVisitFunc(visitFunc visitFunc) VisitOption { + return func(opts *VisitOptions) error { + opts.visitFuncs = append(opts.visitFuncs, visitFunc) + return nil + } +} + +// WithAccessFunc sets the accessFunc for accessing the package. +func WithAccessFunc(accessFunc accessFunc) VisitOption { + return func(opts *VisitOptions) error { + opts.accessFunc = &accessFunc + return nil + } +} + +// WithSource sets the source of the package to be visited. +func WithVisitSource(source *downloader.Source) VisitOption { + return func(opts *VisitOptions) error { + opts.Source = source + return nil + } +} + // Visitor is the interface for visiting a package which is a local path, a remote git/oci path, or a local tar path. type Visitor interface { - Visit(s *downloader.Source, v visitFunc) error + Visit(options ...VisitOption) error } // PkgVisitor is the visitor for visiting a local package. @@ -33,23 +73,57 @@ func NewPkgVisitor(kpmcli *KpmClient) *PkgVisitor { } // Visit visits a local package. -func (pv *PkgVisitor) Visit(s *downloader.Source, v visitFunc) error { +func (pv *PkgVisitor) Visit(options ...VisitOption) error { + opts := &VisitOptions{} + for _, option := range options { + if err := option(opts); err != nil { + return err + } + } + + s := opts.Source + if s == nil { + return fmt.Errorf("source is nil") + } + if !s.IsLocalPath() { return fmt.Errorf("source is not local") } - // Find the root path of the source. - // There must be a kcl.mod file in the root path. - rootPath, err := s.FindRootPath() - if err != nil { - return err + + var accessFunc accessFunc + var kclPkg *pkg.KclPkg + if opts.accessFunc == nil || *opts.accessFunc == nil { + accessFunc = func(source *downloader.Source) (*pkg.KclPkg, error) { + // Find the root path of the source. + // There must be a kcl.mod file in the root path. + rootPath, err := s.FindRootPath() + if err != nil { + return nil, err + } + + kclPkg, err = pv.kpmcli.LoadPkgFromPath(rootPath) + if err != nil { + return nil, err + } + return kclPkg, nil + } + } else { + accessFunc = *opts.accessFunc } - kclPkg, err := pv.kpmcli.LoadPkgFromPath(rootPath) + kclPkg, err := accessFunc(s) if err != nil { return err } - return v(kclPkg) + for _, visitFunc := range opts.visitFuncs { + err = visitFunc(kclPkg) + if err != nil { + return err + } + } + + return nil } // VirtualPkgVisitor is the visitor for visiting a package which do not have a kcl.mod file in the root path. @@ -67,24 +141,55 @@ func NewVirtualPkgVisitor(pv *PkgVisitor) *VirtualPkgVisitor { // Visit visits a package which do not have a kcl.mod file in the root path. // It will create a virtual kcl.mod file in the root path. // And then kcl.mod file will be cleaned after the visitFunc is executed. -func (vpv *VirtualPkgVisitor) Visit(s *downloader.Source, v visitFunc) error { +func (vpv *VirtualPkgVisitor) Visit(options ...VisitOption) error { + opts := &VisitOptions{} + for _, option := range options { + if err := option(opts); err != nil { + return err + } + } + + s := opts.Source + if s == nil { + return fmt.Errorf("source is nil") + } + if !s.IsLocalPath() { return fmt.Errorf("source is not local") } - sourcePath, err := s.ToFilePath() + var accessFunc accessFunc + var kclPkg *pkg.KclPkg + if opts.accessFunc == nil || *opts.accessFunc == nil { + accessFunc = func(source *downloader.Source) (*pkg.KclPkg, error) { + sourcePath, err := s.ToFilePath() + if err != nil { + return nil, err + } + initOpts := opt.InitOptions{ + Name: "vPkg_" + uuid.New().String(), + InitPath: sourcePath, + } + kpkg := pkg.NewKclPkg(&initOpts) + return &kpkg, nil + } + } else { + accessFunc = *opts.accessFunc + } + + kclPkg, err := accessFunc(s) if err != nil { return err } - initOpts := opt.InitOptions{ - Name: "vPkg_" + uuid.New().String(), - InitPath: sourcePath, + for _, visitFunc := range opts.visitFuncs { + err = visitFunc(kclPkg) + if err != nil { + return err + } } - kpkg := pkg.NewKclPkg(&initOpts) - // If the required files are present, proceed with the visitFunc - return v(&kpkg) + return nil } // RemoteVisitor is the visitor for visiting a remote package. @@ -104,11 +209,25 @@ func NewRemoteVisitor(pv *PkgVisitor) *RemoteVisitor { // Visit visits a remote package. // It will download the remote package to a temporary directory. // And the tmp directory will be cleaned after the visitFunc is executed. -func (rv *RemoteVisitor) Visit(s *downloader.Source, v visitFunc) error { +func (rv *RemoteVisitor) Visit(options ...VisitOption) error { + opts := &VisitOptions{} + for _, option := range options { + if err := option(opts); err != nil { + return err + } + } + + s := opts.Source + if s == nil { + return fmt.Errorf("source is nil") + } + if !s.IsRemote() { return fmt.Errorf("source is not remote") } + var accessFunc accessFunc + var kclPkg *pkg.KclPkg tmpDir, err := os.MkdirTemp("", "") if err != nil { return err @@ -117,41 +236,61 @@ func (rv *RemoteVisitor) Visit(s *downloader.Source, v visitFunc) error { if s.Git != nil { tmpDir = filepath.Join(tmpDir, constants.GitScheme) } + defer os.RemoveAll(tmpDir) - credCli, err := rv.kpmcli.GetCredsClient() - if err != nil { - return err + if opts.accessFunc == nil || *opts.accessFunc == nil { + accessFunc = func(source *downloader.Source) (*pkg.KclPkg, error) { + + credCli, err := rv.kpmcli.GetCredsClient() + if err != nil { + return nil, err + } + + err = rv.kpmcli.DepDownloader.Download(*downloader.NewDownloadOptions( + downloader.WithLocalPath(tmpDir), + downloader.WithSource(*s), + downloader.WithLogWriter(rv.kpmcli.GetLogWriter()), + downloader.WithSettings(*rv.kpmcli.GetSettings()), + downloader.WithCredsClient(credCli), + downloader.WithCachePath(rv.CachePath), + downloader.WithEnableCache(rv.EnableCache), + downloader.WithInsecureSkipTLSverify(rv.kpmcli.insecureSkipTLSverify), + )) + + if err != nil { + return nil, err + } + pkgPath := tmpDir + if s.Git != nil && len(s.Git.Package) > 0 { + pkgPath, err = utils.FindPackage(tmpDir, s.Git.Package) + if err != nil { + return nil, err + } + } + + kclPkg, err := rv.kpmcli.LoadPkgFromPath(pkgPath) + if err != nil { + return nil, err + } + return kclPkg, nil + } + } else { + accessFunc = *opts.accessFunc } - defer os.RemoveAll(tmpDir) - err = rv.kpmcli.DepDownloader.Download(*downloader.NewDownloadOptions( - downloader.WithLocalPath(tmpDir), - downloader.WithSource(*s), - downloader.WithLogWriter(rv.kpmcli.GetLogWriter()), - downloader.WithSettings(*rv.kpmcli.GetSettings()), - downloader.WithCredsClient(credCli), - downloader.WithCachePath(rv.CachePath), - downloader.WithEnableCache(rv.EnableCache), - downloader.WithInsecureSkipTLSverify(rv.kpmcli.insecureSkipTLSverify), - )) - + kclPkg, err = accessFunc(s) if err != nil { return err } - pkgPath := tmpDir - if s.Git != nil && len(s.Git.Package) > 0 { - pkgPath, err = utils.FindPackage(tmpDir, s.Git.Package) + + for _, visitFunc := range opts.visitFuncs { + err = visitFunc(kclPkg) if err != nil { return err } } - kclPkg, err := rv.kpmcli.LoadPkgFromPath(pkgPath) - if err != nil { - return err - } - - return v(kclPkg) + return nil } // ArchiveVisitor is the visitor for visiting a package which is a local tar/tgz path. @@ -193,32 +332,65 @@ func (av *ArchiveVisitor) extractArchiveFromSource(s *downloader.Source, extract // Visit visits a package which is a local tar/tgz path. // It will extract the archive file to a temporary directory. // And the tmp directory will be cleaned after the visitFunc is executed. -func (av *ArchiveVisitor) Visit(s *downloader.Source, v visitFunc) error { +func (av *ArchiveVisitor) Visit(options ...VisitOption) error { + + opts := &VisitOptions{} + for _, option := range options { + if err := option(opts); err != nil { + return err + } + } + + s := opts.Source + if s == nil { + return fmt.Errorf("source is nil") + } if !s.IsLocalTarPath() && !s.IsLocalTgzPath() { return fmt.Errorf("source is not local tar path") } - tmpDir, err := os.MkdirTemp("", "") - if err != nil { - return err - } + var accessFunc accessFunc + var kclPkg *pkg.KclPkg + if opts.accessFunc == nil || *opts.accessFunc == nil { + accessFunc = func(source *downloader.Source) (*pkg.KclPkg, error) { + tmpDir, err := os.MkdirTemp("", "") + if err != nil { + return nil, err + } - defer os.RemoveAll(tmpDir) + defer os.RemoveAll(tmpDir) - err = av.extractArchiveFromSource(s, tmpDir) + err = av.extractArchiveFromSource(s, tmpDir) - if err != nil { - return err - } + if err != nil { + return nil, err + } + + kclPkg, err := av.kpmcli.LoadPkgFromPath(tmpDir) - kclPkg, err := av.kpmcli.LoadPkgFromPath(tmpDir) + if err != nil { + return nil, err + } + return kclPkg, nil + } + } else { + accessFunc = *opts.accessFunc + } + kclPkg, err := accessFunc(s) if err != nil { return err } - return v(kclPkg) + for _, visitFunc := range opts.visitFuncs { + err = visitFunc(kclPkg) + if err != nil { + return err + } + } + + return nil } // NewVisitor is a factory function to create a new Visitor. diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index 87deaa47..2d9d486e 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -275,9 +275,6 @@ func (d *Dependency) GenPathSuffix() string { // TODO: new local dependency structure will replace this // issue: https://github.com/kcl-lang/kpm/issues/384 name := strings.TrimSuffix(filepath.Base(d.Source.Git.Url), filepath.Ext(d.Source.Git.Url)) - if d.Source.Git.GetPackage() != "" { - name = strings.Split(d.FullName, "_")[0] - } if len(d.Source.Git.Tag) != 0 { storePkgName = fmt.Sprintf(PKG_NAME_PATTERN, name, d.Source.Git.Tag) } else if len(d.Source.Git.Commit) != 0 { From a4e17c51ce36233135f8696a297640f35e8044c3 Mon Sep 17 00:00:00 2001 From: zongz Date: Thu, 12 Sep 2024 10:38:59 +0800 Subject: [PATCH 3/6] fix: fix windows path Signed-off-by: zongz --- pkg/client/client_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 1f1f4811..7a802084 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -853,7 +853,11 @@ func TestResolveMetadataInJsonStr(t *testing.T) { assert.Equal(t, utils.DirExists(vendorDir), false) assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "flask-demo-kcl-manifests_ade147b")), false) assert.Equal(t, err, nil) - expectedStr := "{\"packages\":{\"flask_demo_kcl_manifests\":{\"name\":\"flask_demo_kcl_manifests\",\"manifest_path\":\"not_exist/flask-demo-kcl-manifests_ade147b\"}}}" + + expectedStr := fmt.Sprintf( + "{\"packages\":{\"flask_demo_kcl_manifests\":{\"name\":\"flask_demo_kcl_manifests\",\"manifest_path\":\"%s\"}}}", + filepath.Join("not_exist", "flask-demo-kcl-manifests_ade147b"), + ) assert.Equal(t, res, expectedStr) defer func() { if r := os.RemoveAll(filepath.Join("not_exist", "flask-demo-kcl-manifests_ade147b")); r != nil { From 593b529a5c02ecb25bcc1c535d7911133225c73f Mon Sep 17 00:00:00 2001 From: zongz Date: Thu, 12 Sep 2024 10:49:44 +0800 Subject: [PATCH 4/6] fix: rm useless changes Signed-off-by: zongz --- pkg/client/client.go | 19 +------------------ pkg/client/resolver.go | 1 - 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index c767d0a6..d902eda5 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -324,6 +324,7 @@ func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) erro return nil } +// Deprecated: Use `KpmClient.Update()` instead. func (c *KpmClient) resolvePkgDeps(kclPkg *pkg.KclPkg, lockDeps *pkg.Dependencies, update bool) error { var searchPath string kclPkg.NoSumCheck = c.noSumCheck @@ -441,17 +442,6 @@ func (c *KpmClient) resolvePkgDeps(kclPkg *pkg.KclPkg, lockDeps *pkg.Dependencie } } - if update { - // Update the dependencies and select the version by mvs. - kclPkg.NoSumCheck = c.noSumCheck - _, err := c.Update( - WithUpdatedKclPkg(kclPkg), - ) - if err != nil { - return err - } - } - return nil } @@ -924,13 +914,6 @@ func (c *KpmClient) vendorDeps(kclPkg *pkg.KclPkg, vendorPath string) error { // If the package already exists in the 'vendor', do nothing. if utils.DirExists(vendorFullPath) { - if d.GetPackage() != "" { - tempVendorFullPath, err := utils.FindPackage(vendorFullPath, d.GetPackage()) - if err != nil { - return err - } - vendorFullPath = tempVendorFullPath - } d.LocalFullPath = vendorFullPath lockDeps[i] = d continue diff --git a/pkg/client/resolver.go b/pkg/client/resolver.go index 5a9e8c94..7449263c 100644 --- a/pkg/client/resolver.go +++ b/pkg/client/resolver.go @@ -21,7 +21,6 @@ type ResolveOption func(*ResolveOptions) error type resolveFunc func(dep *pkg.Dependency, parentPkg *pkg.KclPkg) error type ResolveOptions struct { - // Source is the source of the package to be resolved. kpkg *pkg.KclPkg // EnableCache is the flag to enable the cache during the resolving the remote package. From 0e6aa964e1e3a69203991e12af618f3839318ecf Mon Sep 17 00:00:00 2001 From: zongz Date: Thu, 12 Sep 2024 11:00:28 +0800 Subject: [PATCH 5/6] fix: fix windows path Signed-off-by: zongz --- pkg/client/client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 7a802084..1c4dc546 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -856,7 +856,7 @@ func TestResolveMetadataInJsonStr(t *testing.T) { expectedStr := fmt.Sprintf( "{\"packages\":{\"flask_demo_kcl_manifests\":{\"name\":\"flask_demo_kcl_manifests\",\"manifest_path\":\"%s\"}}}", - filepath.Join("not_exist", "flask-demo-kcl-manifests_ade147b"), + filepath.ToSlash(filepath.Join("not_exist", "flask-demo-kcl-manifests_ade147b")), ) assert.Equal(t, res, expectedStr) defer func() { @@ -1194,7 +1194,7 @@ func TestMetadataOffline(t *testing.T) { assert.Equal(t, res, "{\"packages\":{}}") content_after_metadata, err := os.ReadFile(kclMod) assert.Equal(t, err, nil) - assert.Equal(t, string(content_after_metadata), string(uglyContent)) + assert.Equal(t, utils.RmNewline(string(content_after_metadata)), utils.RmNewline(string(uglyContent))) res, err = kpmcli.ResolveDepsMetadataInJsonStr(kclPkg, true) assert.Equal(t, err, nil) From c6ef57edffdf2d995eeb3a45b337f8415b0e0b92 Mon Sep 17 00:00:00 2001 From: zongz Date: Thu, 12 Sep 2024 11:12:41 +0800 Subject: [PATCH 6/6] fix: fix windows path Signed-off-by: zongz --- pkg/client/client_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 1c4dc546..1967a3a4 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -854,10 +854,16 @@ func TestResolveMetadataInJsonStr(t *testing.T) { assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "flask-demo-kcl-manifests_ade147b")), false) assert.Equal(t, err, nil) + expectedPath := filepath.Join("not_exist", "flask-demo-kcl-manifests_ade147b") + if runtime.GOOS == "windows" { + expectedPath = strings.ReplaceAll(expectedPath, "\\", "\\\\") + } + expectedStr := fmt.Sprintf( "{\"packages\":{\"flask_demo_kcl_manifests\":{\"name\":\"flask_demo_kcl_manifests\",\"manifest_path\":\"%s\"}}}", - filepath.ToSlash(filepath.Join("not_exist", "flask-demo-kcl-manifests_ade147b")), + expectedPath, ) + assert.Equal(t, res, expectedStr) defer func() { if r := os.RemoveAll(filepath.Join("not_exist", "flask-demo-kcl-manifests_ade147b")); r != nil {