From 299eb4fbeb49fa23ae081c68c5db9076dd0562d4 Mon Sep 17 00:00:00 2001 From: Nitishkumar Singh Date: Sat, 22 Jun 2024 21:19:13 +0530 Subject: [PATCH] include system url as template Signed-off-by: Nitishkumar Singh move powershell and go to tools Signed-off-by: Nitishkumar Singh empty commit Signed-off-by: Nitishkumar Singh implement custom version resolver Signed-off-by: Nitishkumar Singh switch node to tools Signed-off-by: Nitishkumar Singh --- cmd/get.go | 2 +- cmd/system/go.go | 81 ++---------- cmd/system/node.go | 95 ++++---------- cmd/system/powershell.go | 50 ++----- pkg/get/download.go | 62 +++++++++ pkg/get/get.go | 118 ++++------------- pkg/get/get_test.go | 251 +++++++++++++++++++++++++++++++++--- pkg/get/tools.go | 171 ++++++++++++++++++------ pkg/get/url_checker_test.go | 2 + pkg/get/version_resolver.go | 210 ++++++++++++++++++++++++++++++ 10 files changed, 715 insertions(+), 327 deletions(-) create mode 100644 pkg/get/version_resolver.go diff --git a/cmd/get.go b/cmd/get.go index 7d4b23b0a..f61d08e7a 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -24,7 +24,7 @@ import ( // MakeGet creates the Get command to download software func MakeGet() *cobra.Command { - tools := get.MakeTools() + tools := get.MakeToolsWithoutSystemApp() sort.Sort(tools) var validToolOptions []string = make([]string, len(tools)) for _, t := range tools { diff --git a/cmd/system/go.go b/cmd/system/go.go index deed1f799..f731a7e9e 100644 --- a/cmd/system/go.go +++ b/cmd/system/go.go @@ -5,14 +5,10 @@ package system import ( "fmt" - "io" - "net/http" "os" "path" "strings" - "github.com/alexellis/arkade/pkg" - "github.com/alexellis/arkade/pkg/archive" "github.com/alexellis/arkade/pkg/env" "github.com/alexellis/arkade/pkg/get" "github.com/spf13/cobra" @@ -53,51 +49,31 @@ func MakeInstallGo() *cobra.Command { return fmt.Errorf("this app only supports Linux") } - dlArch := arch - if arch == "x86_64" { - dlArch = "amd64" - } else if arch == "aarch64" { - dlArch = "arm64" - } else if arch == "armv7" || arch == "armv7l" { - dlArch = "armv6l" - } - - if len(version) == 0 { - v, err := getGoVersion() - if err != nil { - return err + tools := get.MakeTools() + var tool *get.Tool + for _, t := range tools { + if t.Name == "go" { + tool = &t + break } - - version = v - } else if !strings.HasPrefix(version, "go") { - version = "go" + version } - fmt.Printf("Installing version: %s for: %s\n", version, dlArch) - - dlURL := fmt.Sprintf("https://go.dev/dl/%s.%s-%s.tar.gz", version, strings.ToLower(osVer), dlArch) - fmt.Printf("Downloading from: %s\n", dlURL) + if tool == nil { + return fmt.Errorf("unable to find go definition") + } progress, _ := cmd.Flags().GetBool("progress") - outPath, err := get.DownloadFileP(dlURL, progress) + tempPath, err := get.DownloadNested(tool, arch, osVer, version, installPath, progress, !progress) if err != nil { return err } - defer os.Remove(outPath) - - fmt.Printf("Downloaded to: %s\n", outPath) - f, err := os.OpenFile(outPath, os.O_RDONLY, 0644) + err = get.MoveTo(tempPath, installPath) if err != nil { return err } - defer f.Close() - fmt.Printf("Unpacking Go to: %s\n", path.Join(installPath, "go")) - - if err := archive.UntarNested(f, installPath, true, false); err != nil { - return err - } + fmt.Printf("Downloaded to: %sgo\n", installPath) fmt.Printf("\nexport PATH=$PATH:%s:$HOME/go/bin\n"+ "export GOPATH=$HOME/go/\n", path.Join(installPath, "go", "bin")) @@ -107,36 +83,3 @@ func MakeInstallGo() *cobra.Command { return command } - -func getGoVersion() (string, error) { - req, err := http.NewRequest(http.MethodGet, "https://go.dev/VERSION?m=text", nil) - if err != nil { - return "", err - } - - req.Header.Set("User-Agent", pkg.UserAgent()) - - res, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - - if res.Body == nil { - return "", fmt.Errorf("unexpected empty body") - } - - defer res.Body.Close() - body, _ := io.ReadAll(res.Body) - - if res.StatusCode != http.StatusOK { - return "", fmt.Errorf("unexpected status code: %d", res.StatusCode) - } - - content := strings.TrimSpace(string(body)) - version, _, ok := strings.Cut(content, "\n") - if !ok { - return "", fmt.Errorf("format unexpected: %q", content) - } - - return version, nil -} diff --git a/cmd/system/node.go b/cmd/system/node.go index 8a8e7686f..04177e650 100644 --- a/cmd/system/node.go +++ b/cmd/system/node.go @@ -2,58 +2,14 @@ package system import ( "fmt" - "io" - "net/http" "os" - "regexp" "strings" - "github.com/alexellis/arkade/pkg" - "github.com/alexellis/arkade/pkg/archive" "github.com/alexellis/arkade/pkg/env" "github.com/alexellis/arkade/pkg/get" - cp "github.com/otiai10/copy" "github.com/spf13/cobra" ) -func getLatestNodeVersion(version, channel string) (*string, error) { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://nodejs.org/download/%s/%s", channel, version), nil) - if err != nil { - return nil, err - } - - req.Header.Set("User-Agent", pkg.UserAgent()) - - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - - if res.Body != nil { - defer res.Body.Close() - } - - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("could not find latest version for %s, (%d), body: %s", version, res.StatusCode, string(body)) - } - - regex := regexp.MustCompile(`(?m)node-v(\d+.\d+.\d+)-linux-.*`) - result := regex.FindStringSubmatch(string(body)) - - if len(result) < 2 { - if v, ok := os.LookupEnv("ARK_DEBUG"); ok && v == "1" { - fmt.Printf("Body: %s\n", string(body)) - } - return nil, fmt.Errorf("could not find latest version for %s, (%d), %s", version, res.StatusCode, result) - } - return &result[1], nil -} - func MakeInstallNode() *cobra.Command { command := &cobra.Command{ Use: "node", @@ -94,12 +50,18 @@ func MakeInstallNode() *cobra.Command { dlArch = "arm64" } + resolver := &get.NodeVersionResolver{ + Channel: channel, + Version: version, + } + if (version == "latest" || strings.Contains(version, "latest-")) && channel == "release" { - v, err := getLatestNodeVersion(version, channel) + v, err := resolver.GetVersion() if err != nil { return err } - version = *v + version = v + resolver.Version = v } else if (version == "latest" || strings.Contains(version, "latest-")) && channel == "nightly" { return fmt.Errorf("please set a specific version for downloading a nightly builds") } @@ -108,39 +70,36 @@ func MakeInstallNode() *cobra.Command { version = "v" + version } - fmt.Printf("Installing version: %s for: %s\n", version, dlArch) - filename := fmt.Sprintf("%s/%s.tar.gz", version, fmt.Sprintf("node-%s-linux-%s", version, dlArch)) - dlURL := fmt.Sprintf("https://nodejs.org/download/%s/%s", channel, filename) + tools := get.MakeTools() + var tool *get.Tool + for _, t := range tools { + if t.Name == "node" { + tool = &t + break + } + } - fmt.Printf("Downloading from: %s\n", dlURL) - outPath, err := get.DownloadFileP(dlURL, progress) - if err != nil { - return err + if tool == nil { + return fmt.Errorf("unable to find node definition") } - defer os.Remove(outPath) - fmt.Printf("Downloaded to: %s\n", outPath) - f, err := os.OpenFile(outPath, os.O_RDONLY, 0644) - if err != nil { - return err + tool.VersionResolver = &get.NodeVersionResolver{ + Channel: channel, + Version: version, } - defer f.Close() - tempUnpackPath, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("%s*", "node")) + tempPath, err := get.DownloadNested(tool, arch, osVer, version, installPath, progress, !progress) + defer os.RemoveAll(tempPath) if err != nil { return err } - defer os.RemoveAll(tempUnpackPath) - fmt.Printf("Unpacking binaries to: %s\n", tempUnpackPath) - if err = archive.UntarNested(f, tempUnpackPath, true, false); err != nil { - return err - } + fmt.Printf("Temp Path: %s \n", tempPath) - fmt.Printf("Copying binaries to: %s\n", installPath) - nodeDir := fmt.Sprintf("%s/%s", tempUnpackPath, fmt.Sprintf("node-%s-linux-%s", version, dlArch)) - if err := cp.Copy(nodeDir, installPath); err != nil { + err = get.MoveFromInternalDir(fmt.Sprintf("node-%s-linux-%s", version, dlArch), tempPath, installPath) + if err != nil { return err } + return nil } return command diff --git a/cmd/system/powershell.go b/cmd/system/powershell.go index c6b136014..6cd89a94b 100644 --- a/cmd/system/powershell.go +++ b/cmd/system/powershell.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/Masterminds/semver" - "github.com/alexellis/arkade/pkg/archive" "github.com/alexellis/arkade/pkg/env" "github.com/alexellis/arkade/pkg/get" "github.com/spf13/cobra" @@ -48,17 +47,21 @@ func MakeInstallPowershell() *cobra.Command { arch, _ = cmd.Flags().GetString("arch") } - dlArch := arch - if arch == "x86_64" { - dlArch = "x64" - } else if arch == "aarch64" { - dlArch = "arm64" - } else if arch == "armv7" || arch == "armv7l" { - dlArch = "arm32" + tools := get.MakeTools() + var tool *get.Tool + for _, t := range tools { + if t.Name == "pwsh" { + tool = &t + break + } + } + + if tool == nil { + return fmt.Errorf("unable to find powershell definition") } if version == "" { - v, err := get.FindGitHubRelease("PowerShell", "PowerShell") + v, err := get.FindGitHubRelease(tool.Owner, tool.Repo) if err != nil { return err } @@ -67,14 +70,8 @@ func MakeInstallPowershell() *cobra.Command { version = "v" + version } - fmt.Printf("Installing version: %s for: %s\n", version, dlArch) - semVer := semver.MustParse(version) majorVersion := semVer.Major() - // semVer := strings.TrimPrefix(version, "v") - - // majorVerDemlimiter := strings.Index(semVer, ".") - // majorVersion := semVer[:majorVerDemlimiter] installPath = fmt.Sprintf("%s/%d", installPath, majorVersion) @@ -82,35 +79,16 @@ func MakeInstallPowershell() *cobra.Command { installPath = strings.ReplaceAll(installPath, "$HOME", os.Getenv("HOME")) - if err := os.MkdirAll(installPath, 0755); err != nil && !os.IsExist(err) { - fmt.Printf("Error creating directory %s, error: %s\n", installPath, err.Error()) - } - - filename := fmt.Sprintf("powershell-%s-linux-%s.tar.gz", semVer, dlArch) - dlURL := fmt.Sprintf(githubDownloadTemplate, "PowerShell", "PowerShell", version, filename) - - fmt.Printf("Downloading from: %s\n", dlURL) - progress, _ := cmd.Flags().GetBool("progress") - outPath, err := get.DownloadFileP(dlURL, progress) + tempPath, err := get.DownloadNested(tool, arch, osVer, version, installPath, progress, !progress) if err != nil { return err } - defer os.Remove(outPath) - fmt.Printf("Downloaded to: %s\n", outPath) - - f, err := os.OpenFile(outPath, os.O_RDONLY, 0644) + err = get.MoveTo(tempPath, installPath) if err != nil { return err } - defer f.Close() - - fmt.Printf("Unpacking Powershell to: %s\n", installPath) - - if err := archive.Untar(f, installPath, true, true); err != nil { - return err - } lnPath := "/usr/bin/pwsh" fmt.Printf("Creating Symbolic link to: %s\n", lnPath) diff --git a/pkg/get/download.go b/pkg/get/download.go index b3538de78..7453df654 100644 --- a/pkg/get/download.go +++ b/pkg/get/download.go @@ -15,6 +15,7 @@ import ( "github.com/alexellis/arkade/pkg/config" "github.com/alexellis/arkade/pkg/env" "github.com/cheggaaa/pb/v3" + cp "github.com/otiai10/copy" ) const ( @@ -29,6 +30,67 @@ func (e *ErrNotFound) Error() string { return "server returned status: 404" } +// DownloadNested Returns the temporary location of unarchived directory +func DownloadNested(tool *Tool, arch, operatingSystem, version string, movePath string, displayProgress, quiet bool) (string, error) { + downloadURL, err := GetDownloadURL(tool, + strings.ToLower(operatingSystem), + strings.ToLower(arch), + version, quiet) + if err != nil { + return "", err + } + + if !quiet { + fmt.Printf("Downloading: %s\n", downloadURL) + } + + outPath, err := DownloadFileP(downloadURL, displayProgress) + if err != nil { + return "", err + } + defer os.Remove(outPath) + + fmt.Printf("Downloaded to: %s\n", outPath) + + f, err := os.OpenFile(outPath, os.O_RDONLY, 0644) + if err != nil { + return "", err + } + defer f.Close() + + tempUnpackPath, err := os.MkdirTemp(os.TempDir(), "arkade-*") + if err != nil { + return "", err + } + tempUnpackPath = fmt.Sprintf("%s/%s", tempUnpackPath, tool.Name) + + fmt.Printf("Unpacking %s to: %s\n", tool.Name, tempUnpackPath) + if err = archive.UntarNested(f, tempUnpackPath, true, true); err != nil { + return "", err + } + + return tempUnpackPath, nil +} + +func MoveTo(srcPath, dstPath string) error { + if err := os.MkdirAll(dstPath, 0755); err != nil && !os.IsExist(err) { + fmt.Printf("Error creating directory %s, error: %s\n", dstPath, err.Error()) + } + + fmt.Printf("Copying binaries to: %s\n", dstPath) + opts := cp.Options{ + AddPermission: 0755, + } + if err := cp.Copy(srcPath, dstPath, opts); err != nil { + return err + } + return nil +} + +func MoveFromInternalDir(internalDir, srcPath, dstPath string) error { + return MoveTo(fmt.Sprintf("%s/%s", srcPath, internalDir), dstPath) +} + func Download(tool *Tool, arch, operatingSystem, version string, movePath string, displayProgress, quiet bool) (string, string, error) { downloadURL, err := GetDownloadURL(tool, diff --git a/pkg/get/get.go b/pkg/get/get.go index 16122dc8b..491d0875d 100644 --- a/pkg/get/get.go +++ b/pkg/get/get.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log" + "maps" "net" "net/http" "strings" @@ -17,9 +18,6 @@ import ( "github.com/alexellis/arkade/pkg/env" ) -const GitHubVersionStrategy = "github" -const k8sVersionStrategy = "k8s" - var supportedOS = [...]string{"linux", "darwin", "ming"} var supportedArchitectures = [...]string{"x86_64", "arm", "amd64", "armv6l", "armv7l", "arm64", "aarch64"} @@ -44,9 +42,6 @@ type Tool struct { // if any only if only BinaryTemplate is specified. Version string - // Bespoke approach for finding version when none is set. - VersionStrategy string - // Description of what the tool is used for. Description string @@ -64,6 +59,13 @@ type Tool struct { // NoExtension is required for tooling such as kubectx // which at time of writing is a bash script. NoExtension bool + + // true if tool should be used as system install only + SystemOnly bool + + // Provides custom mechanism to resolve version for different tools + // Bespoke approach for finding version when none is set. + VersionResolver VersionResolver } type ToolLocal struct { @@ -73,6 +75,7 @@ type ToolLocal struct { var templateFuncs = map[string]interface{}{ "HasPrefix": func(s, prefix string) bool { return strings.HasPrefix(s, prefix) }, + "ToLower": func(s string) string { return strings.ToLower(s) }, } func (tool Tool) IsArchive(quiet bool) (bool, error) { @@ -142,30 +145,23 @@ func (tool Tool) Head(uri string) (int, string, http.Header, error) { func (tool Tool) GetURL(os, arch, version string, quiet bool) (string, error) { + var resolver VersionResolver + resolver = tool.VersionResolver + if resolver == nil { + resolver = &GithubVersionResolver{tool.Owner, tool.Repo} + } + if len(version) == 0 { if !quiet { log.Printf("Looking up version for %s", tool.Name) } - if len(tool.URLTemplate) == 0 || - strings.Contains(tool.URLTemplate, "https://github.com/") || - tool.VersionStrategy == GitHubVersionStrategy { - - v, err := FindGitHubRelease(tool.Owner, tool.Repo) - if err != nil { - return "", err - } - version = v - } - - if tool.VersionStrategy == k8sVersionStrategy { - v, err := FindK8sRelease() - if err != nil { - return "", err - } - version = v + v, err := resolver.GetVersion() + if err != nil { + return "", err } + version = v if !quiet { log.Printf("Found: %s", version) @@ -173,7 +169,7 @@ func (tool Tool) GetURL(os, arch, version string, quiet bool) (string, error) { } if len(tool.URLTemplate) > 0 { - return getByDownloadTemplate(tool, os, arch, version) + return getByDownloadTemplate(tool, os, arch, version, resolver.Inputs()) } return getURLByGithubTemplate(tool, os, arch, version) @@ -209,75 +205,8 @@ func getURLByGithubTemplate(tool Tool, os, arch, version string) (string, error) } func FindGitHubRelease(owner, repo string) (string, error) { - url := fmt.Sprintf("https://github.com/%s/%s/releases/latest", owner, repo) - - client := makeHTTPClient(&githubTimeout, false) - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - } - - req, err := http.NewRequest(http.MethodHead, url, nil) - if err != nil { - return "", err - } - - req.Header.Set("User-Agent", pkg.UserAgent()) - - res, err := client.Do(req) - if err != nil { - return "", err - } - - if res.Body != nil { - defer res.Body.Close() - } - - if res.StatusCode != http.StatusMovedPermanently && res.StatusCode != http.StatusFound { - return "", fmt.Errorf("server returned status: %d", res.StatusCode) - } - - loc := res.Header.Get("Location") - if len(loc) == 0 { - return "", fmt.Errorf("unable to determine release of tool") - } - - version := loc[strings.LastIndex(loc, "/")+1:] - return version, nil -} - -func FindK8sRelease() (string, error) { - url := "https://cdn.dl.k8s.io/release/stable.txt" - - timeout := time.Second * 5 - client := makeHTTPClient(&timeout, false) - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - } - - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return "", err - } - - req.Header.Set("User-Agent", pkg.UserAgent()) - - res, err := client.Do(req) - if err != nil { - return "", err - } - - if res.Body == nil { - return "", fmt.Errorf("unable to determine release of tool") - } - - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return "", err - } - - version := string(bodyBytes) - return version, nil + resolver := &GithubVersionResolver{Owner: owner, Repo: repo} + return resolver.GetVersion() } func getBinaryURL(owner, repo, version, downloadName string) string { @@ -291,7 +220,7 @@ func getBinaryURL(owner, repo, version, downloadName string) string { owner, repo, version, downloadName) } -func getByDownloadTemplate(tool Tool, os, arch, version string) (string, error) { +func getByDownloadTemplate(tool Tool, os, arch, version string, resolverInputs map[string]string) (string, error) { var err error t := template.New(tool.Name) t = t.Funcs(templateFuncs) @@ -310,6 +239,7 @@ func getByDownloadTemplate(tool Tool, os, arch, version string) (string, error) "Owner": tool.Owner, "Name": tool.Name, } + maps.Copy(inputs, resolverInputs) if err := t.Execute(&buf, inputs); err != nil { return "", err diff --git a/pkg/get/get_test.go b/pkg/get/get_test.go index 19921efd3..029a80ef7 100644 --- a/pkg/get/get_test.go +++ b/pkg/get/get_test.go @@ -6922,12 +6922,6 @@ func Test_Download1Password(t *testing.T) { version: toolVersion, url: "https://cache.agilebits.com/dist/1P/op2/pkg/v2.17.0/op_linux_amd64_v2.17.0.zip", }, - { - os: "linux", - arch: arch64bit, - version: "", - url: "https://cache.agilebits.com/dist/1P/op2/pkg/v2.17.0/op_linux_amd64_v2.17.0.zip", - }, { os: "linux", arch: archARM7, @@ -7658,19 +7652,6 @@ func Test_DownloadKeploy(t *testing.T) { tool := getTool(name, tools) const toolVersion = "v2.3.0" - // keploy_darwin_all.tar.gz - // 20.4 MB - // 1 hour ago - // keploy_linux_amd64.tar.gz - // 12.4 MB - // 1 hour ago - // keploy_linux_arm64.tar.gz - // 11.4 MB - // 1 hour ago - // keploy_windows_amd64.tar.gz - // 10.4 MB - // 1 hour ago - // keploy_windows_arm64.tar.gz tests := []test{ { @@ -7806,6 +7787,7 @@ func Test_Download_labctl(t *testing.T) { url: "https://github.com/iximiuz/labctl/releases/download/v0.1.8/labctl_linux_amd64.tar.gz", }, } + for _, tc := range tests { got, err := tool.GetURL(tc.os, tc.arch, tc.version, false) if err != nil { @@ -7815,4 +7797,235 @@ func Test_Download_labctl(t *testing.T) { t.Fatalf("\nwant: %s\ngot: %s", tc.url, got) } } + +} + +func Test_DownloadPowershell(t *testing.T) { + tools := MakeTools() + name := "pwsh" + + tool := getTool(name, tools) + + const toolVersion = "v7.4.3" + + tests := []test{ + { + os: "linux", + arch: arch64bit, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-linux-x64.tar.gz", + }, + { + os: "linux", + arch: archARM64, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-linux-arm64.tar.gz", + }, + { + os: "linux", + arch: "armv6l", + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-linux-arm32.tar.gz", + }, + { + os: "linux", + arch: archARM7, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-linux-arm32.tar.gz", + }, + { + os: "darwin", + arch: arch64bit, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-osx-x64.tar.gz", + }, + { + os: "darwin", + arch: archDarwinARM64, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-osx-arm64.tar.gz", + }, + { + os: "ming", + arch: archARM64, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/PowerShell-7.4.3-win-arm64.zip", + }, + { + os: "ming", + arch: arch64bit, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/PowerShell-7.4.3-win-x64.zip", + }, + } + + for _, tc := range tests { + got, err := tool.GetURL(tc.os, tc.arch, tc.version, false) + if err != nil { + t.Fatal(err) + } + if got != tc.url { + t.Errorf("\nwant: %s, \n got: %s", tc.url, got) + } + } + +} + +func Test_DownloadNode(t *testing.T) { + tools := MakeTools() + name := "node" + + tool := getTool(name, tools) + + const toolVersion = "v22.8.0" + + tests := []test{ + { + os: "linux", + arch: arch64bit, + version: toolVersion, + url: "https://nodejs.org/download/release/v22.8.0/node-v22.8.0-linux-x64.tar.gz", + }, + { + os: "linux", + arch: archARM64, + version: toolVersion, + url: "https://nodejs.org/download/release/v22.8.0/node-v22.8.0-linux-arm64.tar.gz", + }, + { + os: "linux", + arch: archARM7, + version: toolVersion, + url: "https://nodejs.org/download/release/v22.8.0/node-v22.8.0-linux-armv7l.tar.gz", + }, + { + os: "darwin", + arch: arch64bit, + version: toolVersion, + url: "https://nodejs.org/download/release/v22.8.0/node-v22.8.0-darwin-x64.tar.gz", + }, + { + os: "darwin", + arch: archDarwinARM64, + version: toolVersion, + url: "https://nodejs.org/download/release/v22.8.0/node-v22.8.0-darwin-arm64.tar.gz", + }, + { + os: "ming", + arch: archARM64, + version: toolVersion, + url: "https://nodejs.org/download/release/v22.8.0/node-v22.8.0-win-arm64.zip", + }, + { + os: "ming", + arch: arch64bit, + version: toolVersion, + url: "https://nodejs.org/download/release/v22.8.0/node-v22.8.0-win-x64.zip", + }, + } + + for _, tc := range tests { + got, err := tool.GetURL(tc.os, tc.arch, tc.version, false) + if err != nil { + t.Fatal(err) + } + if got != tc.url { + t.Errorf("\nwant: %s, \n got: %s", tc.url, got) + } + } + +} + +func Test_DownloadGo(t *testing.T) { + tools := MakeTools() + name := "go" + + tool := getTool(name, tools) + + const toolVersion = "1.22.4" + + tests := []test{ + { + os: "linux", + arch: arch64bit, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.linux-amd64.tar.gz", + }, + { + os: "linux", + arch: archARM64, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.linux-arm64.tar.gz", + }, + { + os: "linux", + arch: "armv6l", + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.linux-armv6l.tar.gz", + }, + { + os: "linux", + arch: archARM7, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.linux-armv6l.tar.gz", + }, + { + os: "darwin", + arch: arch64bit, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.darwin-amd64.tar.gz", + }, + { + os: "darwin", + arch: archARM64, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.darwin-arm64.tar.gz", + }, + { + os: "darwin", + arch: "armv6l", + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.darwin-armv6l.tar.gz", + }, + { + os: "darwin", + arch: archARM7, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.darwin-armv6l.tar.gz", + }, + { + os: "ming", + arch: archARM64, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.windows-arm64.zip", + }, + { + os: "ming", + arch: arch64bit, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.windows-amd64.zip", + }, + { + os: "ming", + arch: "armv6l", + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.windows-armv6l.zip", + }, + { + os: "ming", + arch: archARM7, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.windows-armv6l.zip", + }, + } + for _, tc := range tests { + got, err := tool.GetURL(tc.os, tc.arch, tc.version, false) + if err != nil { + t.Fatal(err) + } + if got != tc.url { + t.Fatalf("\nwant: %s\ngot: %s", tc.url, got) + } + } + } diff --git a/pkg/get/tools.go b/pkg/get/tools.go index a8c83bb38..dd260a26d 100644 --- a/pkg/get/tools.go +++ b/pkg/get/tools.go @@ -21,6 +21,17 @@ func (t Tools) Less(i, j int) bool { type Tools []Tool +func MakeToolsWithoutSystemApp() Tools { + allTools := MakeTools() + tools := []Tool{} + for _, tool := range allTools { + if !tool.SystemOnly { + tools = append(tools, tool) + } + } + return tools +} + func MakeTools() Tools { tools := []Tool{} @@ -51,11 +62,10 @@ func MakeTools() Tools { tools = append(tools, Tool{ - Owner: "helm", - Repo: "helm", - Name: "helm", - VersionStrategy: GitHubVersionStrategy, - Description: "The Kubernetes Package Manager: Think of it like apt/yum/homebrew for Kubernetes.", + Owner: "helm", + Repo: "helm", + Name: "helm", + Description: "The Kubernetes Package Manager: Think of it like apt/yum/homebrew for Kubernetes.", URLTemplate: ` {{$os := .OS}} {{$arch := .Arch}} @@ -153,7 +163,7 @@ func MakeTools() Tools { Owner: "kubernetes", Repo: "kubernetes", Name: "kubectl", - VersionStrategy: k8sVersionStrategy, + VersionResolver: &K8VersionResolver{}, Description: "Run commands against Kubernetes clusters", URLTemplate: `{{$arch := "arm"}} @@ -665,11 +675,10 @@ https://github.com/inlets/inletsctl/releases/download/{{.Version}}/{{$fileName}} tools = append(tools, Tool{ - Owner: "digitalocean", - Repo: "doctl", - Name: "doctl", - VersionStrategy: GitHubVersionStrategy, - Description: "Official command line interface for the DigitalOcean API.", + Owner: "digitalocean", + Repo: "doctl", + Name: "doctl", + Description: "Official command line interface for the DigitalOcean API.", BinaryTemplate: ` {{$osStr := .OS}} {{ if HasPrefix .OS "ming" -}} @@ -842,11 +851,10 @@ https://github.com/inlets/inletsctl/releases/download/{{.Version}}/{{$fileName}} tools = append(tools, Tool{ - Owner: "hashicorp", - Repo: "consul", - Name: "consul", - VersionStrategy: GitHubVersionStrategy, - Description: "A solution to connect and configure applications across dynamic, distributed infrastructure", + Owner: "hashicorp", + Repo: "consul", + Name: "consul", + Description: "A solution to connect and configure applications across dynamic, distributed infrastructure", URLTemplate: ` {{$arch := ""}} {{- if eq .Arch "x86_64" -}} @@ -867,11 +875,10 @@ https://github.com/inlets/inletsctl/releases/download/{{.Version}}/{{$fileName}} tools = append(tools, Tool{ - Owner: "hashicorp", - Repo: "terraform", - Name: "terraform", - VersionStrategy: GitHubVersionStrategy, - Description: "Infrastructure as Code for major cloud providers.", + Owner: "hashicorp", + Repo: "terraform", + Name: "terraform", + Description: "Infrastructure as Code for major cloud providers.", URLTemplate: ` {{$arch := ""}} {{- if eq .Arch "x86_64" -}} @@ -1005,11 +1012,10 @@ https://github.com/inlets/inletsctl/releases/download/{{.Version}}/{{$fileName}} tools = append(tools, Tool{ - Owner: "hashicorp", - Repo: "packer", - Name: "packer", - VersionStrategy: GitHubVersionStrategy, - Description: "Build identical machine images for multiple platforms from a single source configuration.", + Owner: "hashicorp", + Repo: "packer", + Name: "packer", + Description: "Build identical machine images for multiple platforms from a single source configuration.", URLTemplate: ` {{$arch := ""}} {{- if eq .Arch "x86_64" -}} @@ -1030,11 +1036,10 @@ https://github.com/inlets/inletsctl/releases/download/{{.Version}}/{{$fileName}} tools = append(tools, Tool{ - Owner: "hashicorp", - Repo: "waypoint", - Name: "waypoint", - VersionStrategy: GitHubVersionStrategy, - Description: "Easy application deployment for Kubernetes and Amazon ECS", + Owner: "hashicorp", + Repo: "waypoint", + Name: "waypoint", + Description: "Easy application deployment for Kubernetes and Amazon ECS", URLTemplate: ` {{$arch := .Arch}} {{- if eq .Arch "x86_64" -}} @@ -2908,11 +2913,10 @@ https://github.com/{{.Owner}}/{{.Repo}}/releases/download/{{.Version}}/{{.Name}} tools = append(tools, Tool{ - Owner: "hashicorp", - Repo: "vault", - Name: "vault", - VersionStrategy: GitHubVersionStrategy, - Description: "A tool for secrets management, encryption as a service, and privileged access management.", + Owner: "hashicorp", + Repo: "vault", + Name: "vault", + Description: "A tool for secrets management, encryption as a service, and privileged access management.", URLTemplate: ` {{$arch := ""}} {{- if eq .Arch "x86_64" -}} @@ -3842,15 +3846,12 @@ https://github.com/{{.Owner}}/{{.Repo}}/releases/download/{{.Version}}/{{.Name}} Owner: "1password", Name: "op", Description: "1Password CLI enables you to automate administrative tasks and securely provision secrets across development environments.", + Version: "v2.17.0", URLTemplate: ` {{$os := .OS}} {{$arch := .Arch}} {{$version := .Version}} - {{- if eq .Version "" -}} - {{ $version = "v2.17.0" }} - {{- end -}} - {{- if eq .Arch "aarch64" -}} {{ $arch = "arm64" }} {{- else if eq .Arch "x86_64" -}} @@ -4266,5 +4267,95 @@ https://github.com/{{.Owner}}/{{.Repo}}/releases/download/{{.Version}}/{{.Name}} labctl_{{$os}}_{{$arch}}.{{$ext}} `, }) + + tools = append(tools, + Tool{ + Owner: "go", + Repo: "go", + Name: "go", + SystemOnly: true, + VersionResolver: &GoVersionResolver{}, + Description: "Build simple, secure, scalable systems with Go", + URLTemplate: ` + {{$os := .OS}} + {{$arch := .Arch}} + {{$ext := "tar.gz"}} + + {{- if (or (eq .Arch "aarch64") (eq .Arch "arm64")) -}} + {{$arch = "arm64"}} + {{- else if eq .Arch "x86_64" -}} + {{ $arch = "amd64" }} + {{- else if eq .Arch "armv7l" -}} + {{ $arch = "armv6l" }} + {{- end -}} + + {{ if HasPrefix .OS "ming" -}} + {{$os = "windows"}} + {{$ext = "zip"}} + {{- end -}} + + https://{{.Name}}.dev/dl/{{.Name}}{{.VersionNumber}}.{{$os}}-{{$arch}}.{{$ext}}`, + }) + + tools = append(tools, + Tool{ + Owner: "PowerShell", + Repo: "PowerShell", + Name: "pwsh", + SystemOnly: true, + Description: "PowerShell is a cross-platform automation and configuration tool/framework", + URLTemplate: ` + {{$os := .OS}} + {{$arch := .Arch}} + {{$ext := "tar.gz"}} + {{$zipPrefix := ToLower .Repo}} + + {{- if (or (eq .Arch "aarch64") (eq .Arch "arm64")) -}} + {{$arch = "arm64"}} + {{- else if eq .Arch "x86_64" -}} + {{ $arch = "x64" }} + {{- else if (or (eq .Arch "armv6l") (eq .Arch "armv7l")) -}} + {{ $arch = "arm32" }} + {{- end -}} + + {{- if eq .OS "darwin" -}} + {{$os = "osx"}} + {{- else if HasPrefix .OS "ming" -}} + {{$os = "win"}} + {{$ext = "zip"}} + {{$zipPrefix = .Repo}} + {{- end -}} + + https://github.com/{{.Owner}}/{{.Repo}}/releases/download/{{.Version}}/{{$zipPrefix}}-{{.VersionNumber}}-{{$os}}-{{$arch}}.{{$ext}}`, + }) + + tools = append(tools, + Tool{ + Owner: "node", + Repo: "node", + Name: "node", + SystemOnly: true, + VersionResolver: &NodeVersionResolver{ + Channel: "release", + }, + Description: "Node.js is a JavaScript runtime built on Chrome's V8 JavaScript engine", + URLTemplate: ` + {{$os := .OS}} + {{$arch := .Arch}} + {{$ext := "tar.gz"}} + + {{- if (or (eq .Arch "aarch64") (eq .Arch "arm64")) -}} + {{$arch = "arm64"}} + {{- else if eq .Arch "x86_64" -}} + {{ $arch = "x64" }} + {{- end -}} + + {{ if HasPrefix .OS "ming" -}} + {{$os = "win"}} + {{$ext = "zip"}} + {{- end -}} + + https://nodejs.org/download/{{.Channel}}/{{.Version}}/node-{{.Version}}-{{$os}}-{{$arch}}.{{$ext}}`, + }) return tools } diff --git a/pkg/get/url_checker_test.go b/pkg/get/url_checker_test.go index 7679c4235..628e97e20 100644 --- a/pkg/get/url_checker_test.go +++ b/pkg/get/url_checker_test.go @@ -14,6 +14,8 @@ func Test_CheckTools(t *testing.T) { tools := MakeTools() toolsToSkip := []string{ "kumactl", // S3 bucket disallow HEAD requests + // https://github.com/vladimirvivien/ktop/issues/46 + "ktop", // latest release does not include binary } os := "linux" diff --git a/pkg/get/version_resolver.go b/pkg/get/version_resolver.go new file mode 100644 index 000000000..2b2519328 --- /dev/null +++ b/pkg/get/version_resolver.go @@ -0,0 +1,210 @@ +package get + +import ( + "fmt" + "io" + "net/http" + "os" + "regexp" + "strings" + "time" + + "github.com/alexellis/arkade/pkg" +) + +// Tools can implement version resolver to resolve versions differently +type VersionResolver interface { + GetVersion() (string, error) + Inputs() map[string]string +} + +var _ VersionResolver = (*GithubVersionResolver)(nil) + +type GithubVersionResolver struct { + Owner string + Repo string +} + +// Inputs implements VersionResolver. +func (r *GithubVersionResolver) Inputs() map[string]string { + return map[string]string{} +} + +func (r *GithubVersionResolver) GetVersion() (string, error) { + url := fmt.Sprintf("https://github.com/%s/%s/releases/latest", r.Owner, r.Repo) + client := makeHTTPClient(&githubTimeout, false) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + + req, err := http.NewRequest(http.MethodHead, url, nil) + if err != nil { + return "", err + } + + req.Header.Set("User-Agent", pkg.UserAgent()) + + res, err := client.Do(req) + if err != nil { + return "", err + } + + if res.Body != nil { + defer res.Body.Close() + } + + if res.StatusCode != http.StatusMovedPermanently && res.StatusCode != http.StatusFound { + return "", fmt.Errorf("server returned status: %d", res.StatusCode) + } + + loc := res.Header.Get("Location") + if len(loc) == 0 { + return "", fmt.Errorf("unable to determine release of tool") + } + + version := loc[strings.LastIndex(loc, "/")+1:] + return version, nil +} + +var _ VersionResolver = (*K8VersionResolver)(nil) + +type K8VersionResolver struct{} + +// Inputs implements VersionResolver. +func (r *K8VersionResolver) Inputs() map[string]string { + return map[string]string{} +} + +func (r *K8VersionResolver) GetVersion() (string, error) { + url := "https://cdn.dl.k8s.io/release/stable.txt" + + timeout := time.Second * 5 + client := makeHTTPClient(&timeout, false) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return "", err + } + + req.Header.Set("User-Agent", pkg.UserAgent()) + + res, err := client.Do(req) + if err != nil { + return "", err + } + + if res.Body == nil { + return "", fmt.Errorf("unable to determine release of tool") + } + + defer res.Body.Close() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return "", err + } + + version := string(bodyBytes) + return version, nil +} + +var _ VersionResolver = (*GoVersionResolver)(nil) + +type GoVersionResolver struct{} + +// Inputs implements VersionResolver. +func (r *GoVersionResolver) Inputs() map[string]string { + return map[string]string{} +} + +func (r *GoVersionResolver) GetVersion() (string, error) { + req, err := http.NewRequest(http.MethodGet, "https://go.dev/VERSION?m=text", nil) + if err != nil { + return "", err + } + + req.Header.Set("User-Agent", pkg.UserAgent()) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + + if res.Body == nil { + return "", fmt.Errorf("unexpected empty body") + } + + defer res.Body.Close() + body, _ := io.ReadAll(res.Body) + + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + + content := strings.TrimSpace(string(body)) + exp, err := regexp.Compile(`^go(\d+.\d+.\d+)`) + if err != nil { + return "", err + } + + version := exp.FindStringSubmatch(content) + if len(version) < 2 { + return "", fmt.Errorf("failed to fetch go latest version number") + } + + return version[1], nil +} + +var _ VersionResolver = (*NodeVersionResolver)(nil) + +type NodeVersionResolver struct { + Channel string + Version string +} + +// Inputs implements VersionResolver. +func (n *NodeVersionResolver) Inputs() map[string]string { + return map[string]string{ + "Channel": n.Channel, + } +} + +func (n *NodeVersionResolver) GetVersion() (string, error) { + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://nodejs.org/download/%s/%s", n.Channel, n.Version), nil) + if err != nil { + return "", err + } + + req.Header.Set("User-Agent", pkg.UserAgent()) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + + if res.Body != nil { + defer res.Body.Close() + } + + body, err := io.ReadAll(res.Body) + if err != nil { + return "", err + } + + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("could not find latest version for %s, (%d), body: %s", n.Version, res.StatusCode, string(body)) + } + + regex := regexp.MustCompile(`(?m)node-v(\d+.\d+.\d+)-linux-.*`) + result := regex.FindStringSubmatch(string(body)) + + if len(result) < 2 { + if v, ok := os.LookupEnv("ARK_DEBUG"); ok && v == "1" { + fmt.Printf("Body: %s\n", string(body)) + } + return "", fmt.Errorf("could not find latest version for %s, (%d), %s", n.Version, res.StatusCode, result) + } + return result[1], nil +}