From d131a52d29758159c107b7c4c3c75242b438201c Mon Sep 17 00:00:00 2001 From: Daniel Kuffner Date: Sun, 24 May 2015 17:41:43 -0400 Subject: [PATCH] cleanup --- README.md | 33 ++++++----- artifact.go | 34 ++++++++++-- config.go | 14 +++++ download.go | 4 ++ downloader.go | 61 +++++++++++++++++++++ downloader_test.go | 47 ++++++++++++++++ gomaven.go | 134 --------------------------------------------- pom.go | 85 ++++++++++++++++++++++++++++ pom_test.go | 52 ++++++++++++++++++ repo.go | 83 ++++++++++++++++++++++++++-- tracker.go | 99 +++++++++++++++++++++++++++++++++ tracker_test.go | 41 ++++++++++++++ utils.go | 14 +++++ utils_test.go | 17 ++++++ 14 files changed, 557 insertions(+), 161 deletions(-) create mode 100644 config.go create mode 100644 downloader.go create mode 100644 downloader_test.go delete mode 100644 gomaven.go create mode 100644 pom.go create mode 100644 pom_test.go create mode 100644 tracker.go create mode 100644 tracker_test.go create mode 100644 utils.go create mode 100644 utils_test.go diff --git a/README.md b/README.md index 1193931..d275624 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,26 @@ -# gomaven - -gomaven is a command line tool to download artifacts from a maven or nexus repository. +# repo +repo is a command line tool to download artifacts from a maven or nexus repository. ## Get started -Download the precompiled binary for your platform here: https://github.com/chilicat/gomaven/releases/tag/v0.1.0 +Download the precompiled binary for your platform here: https://github.com/chilicat/repo/releases/tag/v0.1.0 -gomaven has a self self-documented command line interface. The best way to figure out what you can do with the tool is calling: +repo has a self self-documented command line interface. The best way to figure out what you can do with the tool is calling: ``` -gomaven help +repo help NAME: - gomaven - to deal with maven repositories + repo - to deal with maven repositories USAGE: - gomaven [global options] command [command options] [arguments...] + repo [global options] command [command options] [arguments...] VERSION: 0.1.0 COMMANDS: - download, d Downloads artifact from maven repository. + pull, d Downloads artifact from maven repository. help, h Shows a list of commands or help for one command GLOBAL OPTIONS: @@ -33,26 +32,26 @@ And from there you can drill into subcommands: ``` -gomaven help download +repo help pull ``` ## Download a artifact -gomaven makes it easy to download artifacts from a maven repository by providing the typical artifact coordinations GROUP_ID:ID:VERSION. The example below downlods the commons-io jar file in version 2.4: +repo makes it easy to download artifacts from a maven repository by providing the typical artifact coordinations GROUP_ID:ID:VERSION. The example below downlods the commons-io jar file in version 2.4: ``` -gomaven download -a commons-io:commons-io:2.4 +repo pull -a commons-io:commons-io:2.4 Download: http://repo1.maven.org/maven2/commons-io/commons-io/2.4/commons-io-2.4.jar -> ./commons-io-2.4.jar ``` -gomaven checks also the md5 checksum of the file before downloading. If you execute the command above a second time you can see that the tool actually skipps the download: +repo checks also the md5 checksum of the file before downloading. If you execute the command above a second time you can see that the tool actually skipps the download: ``` -gomaven download -a commons-io:commons-io:2.4 +repo pull -a commons-io:commons-io:2.4 Download Skipped (up-to-date) -> ./commons-io-2.4.jar ``` @@ -61,7 +60,7 @@ You can also download multiple artifacts: ``` -gomaven download -a -a commons-io:commons-io:2.4,commons-lang:commons-lang:2.4 +repo download -a -a commons-io:commons-io:2.4,commons-lang:commons-lang:2.4 ``` @@ -69,10 +68,10 @@ gomaven download -a -a commons-io:commons-io:2.4,commons-lang:commons-lang:2.4 See help for options: ``` -gomaven download help +repo pull help USAGE: - command download [command options] [arguments...] + command pull [command options] [arguments...] OPTIONS: --artifact, -a Defines artifact coordinates to download (e.g.: commons-io:commons-io:2.4) diff --git a/artifact.go b/artifact.go index 6576ea4..90fd9c6 100644 --- a/artifact.go +++ b/artifact.go @@ -5,21 +5,43 @@ import ( "errors" "strings" "text/template" + "fmt" ) type Artifact struct { Group, Id, Version, Class, Ext string } +type ArtifactInfo struct { + artifact Artifact + url string + destFile string +} + +func (p Artifact) String() string { + return fmt.Sprintf("%s:%s:%s:%s:%s", p.Group, p.Id, p.Version, p.Class, p.Ext) +} + +func (a Artifact) Pom() Artifact { + return Artifact { a.Group, a.Id, a.Version, "pom", "pom" } +} + +func (a Artifact) IsPom() bool { + return a.Class == "pom" && a.Ext == "pom" +} + func ParseArtifact(a string) (Artifact, error) { - tokens := strings.Split(a, ":") + first := strings.Split(a, "@") + tokens := strings.Split(first[0], ":") + + ext := "jar" + if len(first) > 1 { ext = first[1] } + if len(tokens) == 3 { - return Artifact{tokens[0], tokens[1], tokens[2], "", "jar"}, nil + return Artifact{tokens[0], tokens[1], tokens[2], "", ext}, nil } else if len(tokens) == 4 { - return Artifact{tokens[0], tokens[1], tokens[2], tokens[3], "jar"}, nil - } else if len(tokens) == 5 { - return Artifact{tokens[0], tokens[1], tokens[2], tokens[3], tokens[4]}, nil - } + return Artifact{tokens[0], tokens[1], tokens[2], tokens[3], ext}, nil + } return Artifact{}, errors.New("Artifact description is insufficent, minium: ::") } diff --git a/config.go b/config.go new file mode 100644 index 0000000..d58f04f --- /dev/null +++ b/config.go @@ -0,0 +1,14 @@ +package main + +type Config struct { + // Repository base url + BaseUrl string + // Destination folder. + Dest string + // File name template + Template string + + DryRun bool + + WorkersCount int +} \ No newline at end of file diff --git a/download.go b/download.go index 1caf523..c56f624 100644 --- a/download.go +++ b/download.go @@ -47,6 +47,10 @@ func downloadMd5(uri string) (string, error) { return strings.Fields(c)[0], nil } +func DownloadAsString(uri string) (string, error) { + return downloadAsString(uri) +} + func downloadAsString(uri string) (string, error) { resp, err := http.Get(uri) if err != nil { diff --git a/downloader.go b/downloader.go new file mode 100644 index 0000000..042fa1b --- /dev/null +++ b/downloader.go @@ -0,0 +1,61 @@ +package main + +import ( + "path" +) + +type DownloadRequest struct { + artifact Artifact + inMemory bool + consumer chan DownloadResult +} + +type DownloadResult struct { + info ArtifactInfo + err error + content string +} + +type Downloader struct { + num int + conf Config + request chan DownloadRequest +} + +func (d Downloader) Request(r DownloadRequest) { + d.request <- r +} + +func NewDownloader(conf Config) Downloader { + num := conf.WorkersCount + request := make(chan DownloadRequest, num) + d := Downloader { num, conf, request } + for i := 0; i %s", p.GroupId, p.ArtifactId, p.Version, p.Dependencies) +} + +func (p PomDependencies) byScope(scope string) []PomDependency { + found := make([]PomDependency, 0) + for _,d := range p.Dependency { + if d.Scope == scope { + found = append(found, d) + } + } + return found +} + +func (p PomDependencies) ToArtifacts(scope string) []Artifact { + deps := p.byScope(scope) + list := make([]Artifact, len(deps), len(deps)) + for i,dep := range deps { + list[i] = toArtifact(dep) + } + return list +} + +func (p PomDependencies) ToArtifactsAll() []Artifact { + deps := p.Dependency + list := make([]Artifact, len(deps), len(deps)) + for i,dep := range deps { + list[i] = toArtifact(dep) + } + return list +} + + +func toArtifact(dep PomDependency) Artifact { + return Artifact{ dep.GroupId, dep.ArtifactId, dep.Version, "", "jar"} +} + +func ParsePomFromString(xmlStr string) (error, Pom) { + v := Pom{} + err := xml.NewDecoder(strings.NewReader(xmlStr)).Decode(&v) + return err, v +} + +func ParsePomFromFile(file string) (error, Pom) { + v := Pom{} + xmlFile, err := os.Open(file) + if err == nil { + defer xmlFile.Close() + err = xml.NewDecoder(xmlFile).Decode(&v) + } + return err, v +} \ No newline at end of file diff --git a/pom_test.go b/pom_test.go new file mode 100644 index 0000000..cb5698d --- /dev/null +++ b/pom_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "testing" +) + +func TestPomParse(t *testing.T) { + data := ` + + + 4.0.0 + org.hibernate + hibernate-core + 4.3.9.Final + + + org.jboss.logging + jboss-logging + 3.1.3.GA + compile + + + org.jboss.logging + jboss-logging-annotations + 1.2.0.Beta1 + compile + + + + ` + + err, v := ParsePomFromString(data) + if err != nil { + t.Errorf("Parsing failed: %q", err) + } + + asset(t, "org.hibernate", v.GroupId, "Pom.Group") + asset(t, "hibernate-core", v.ArtifactId, "Pom.ArtifactId") + asset(t, "4.3.9.Final", v.Version, "Pom.Version") + + art := v.Dependencies.ToArtifacts("compile") + + asset(t, 2, len(art), "Dependency count") + + logging := art[0] + asset(t, "org.jboss.logging", logging.Group, "Group") + asset(t, "jboss-logging", logging.Id, "Id") + asset(t, "3.1.3.GA", logging.Version, "Version") +} + + + diff --git a/repo.go b/repo.go index 6738a4a..b0f9605 100644 --- a/repo.go +++ b/repo.go @@ -1,13 +1,88 @@ package main import ( + "fmt" + "github.com/codegangsta/cli" + "os" "strings" ) -func ToPath(a Artifact) string { - return strings.Replace(a.Group, ".", "/", 0) + "/" + a.Id + "/" + a.Version +func main() { + app := cli.NewApp() + app.Name = "repo" + app.Usage = "to deal with maven repositories" + app.Version = "0.1.0" + app.Commands = []cli.Command{ + { + Name: "pull", + ShortName: "d", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "artifact,a", + Usage: "Defines artifact coordinates to download (e.g.: commons-io:commons-io:2.4)", + Value: "", + }, + cli.StringFlag{ + Name: "destination,d", + Usage: "Defines download destination.", + EnvVar: "MVN_DEST", + Value: ".", + }, + cli.StringFlag{ + Name: "baseUrl,b", + Usage: "Defines maven base repo url", + EnvVar: "MVN_BASE_URL", + Value: "http://central.maven.org/maven2", + }, + cli.StringFlag{ + Name: "template,t", + Usage: "Defines file template", + EnvVar: "MVN_FILE_TMPL", + Value: "{{.Id}}-{{.Version}}.{{.Ext}}", + }, + cli.BoolFlag{ + Name: "verbose,v", + Usage: "verbose output.", + }, + }, + Usage: "Downloads artifact from maven repository.", + Action: DownloadCommand, + }, + } + app.Run(os.Args) } -func ToUrl(base string, a Artifact) string { - return base + "/" + ToPath(a) + "/" + a.Id + "-" + a.Version + "." + a.Ext +func prepareArtifact(c *cli.Context) []Artifact { + art := c.String("a") + if art == "" { + fatal("Please define artifact: -a commons-io:commons-io:2.4") + } + if c.Bool("verbose") { + fmt.Println("[INFO] Artifact: " + art) + fmt.Println("[INFO] Destination: " + c.String("d")) + fmt.Println("[INFO] Template: " + c.String("t")) + fmt.Println("[INFO] Base URL: " + c.String("b")) + } + artifacts := make([]Artifact, 0) + for _, el := range strings.Split(art, ",") { + artifact, err := ParseArtifact(el) + checkError(err) + artifacts = append(artifacts, artifact) + } + return artifacts +} + +func DownloadCommand(c *cli.Context) { + conf := Config { + c.String("b"), + c.String("d"), + c.String("t"), + false, + 5, + } + tracker := NewTracker(conf) + for _, a := range prepareArtifact(c) { + tracker.Request(a) + } + tracker.Wait() } diff --git a/tracker.go b/tracker.go new file mode 100644 index 0000000..6735e75 --- /dev/null +++ b/tracker.go @@ -0,0 +1,99 @@ +package main + +import ( + "sync/atomic" +) + +var counter int64 = 0 + +type Tracker struct { + conf Config + wait chan error + request chan Artifact +} + +func (t Tracker) Request(artifact Artifact) { + up() + t.request <- artifact +} + +func (t Tracker) Wait() error { + for err := range t.wait { + return err + } + return nil +} + +func up() int64 { + return atomic.AddInt64(&counter, 1) +} + +func down() int64 { + return atomic.AddInt64(&counter, -1) +} + +func NewTracker(conf Config) Tracker { + wait := make(chan error, 1) + requests := make(chan Artifact, 100) + + pomResults := make(chan DownloadResult, 100) + results := make(chan DownloadResult, 100) + + t := Tracker { conf, wait, requests } + + + downloader := NewDownloader(conf) + + go func() { + for res := range pomResults { + err, pom := ParsePomFromString(res.content) + if err != nil { + wait <- res.err + } + for _, a := range pom.Dependencies.ToArtifacts("compile") { + t.Request(a) + } + + if down() <= 0 { + wait <- nil + } + } + }() + + go func() { + set := make(map[string]bool) + for artifact := range requests { + if !set[artifact.String()] { + set[artifact.String()] = true + if !artifact.IsPom() { + up() + pomArtifact := artifact.Pom() + downloader.Request(DownloadRequest { + pomArtifact, + true, + pomResults, + }) + } + downloader.Request(DownloadRequest { + artifact, + false, + results, + }) + } else { + down() + } + } + }() + + go func() { + for res := range results { + if res.err != nil { + wait <- res.err + } else if down() <= 0 { + wait <- nil + } + } + }() + + return t +} \ No newline at end of file diff --git a/tracker_test.go b/tracker_test.go new file mode 100644 index 0000000..b385724 --- /dev/null +++ b/tracker_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "testing" + "fmt" + "time" +) + +func TestTracker(t *testing.T) { + fmt.Printf("Tracker\n") + conf := Config { + "http://central.maven.org/maven2", + "/tmp/down-test", + "{{.Id}}-{{.Version}}.{{.Ext}}", + true, + 5, + } + + tracker := NewTracker(conf) + tracker.Request(Artifact { "commons-lang", "commons-lang", "2.4", "jar", "jar"}) + tracker.Request(Artifact { "org.hibernate", "hibernate-core", "4.3.9.Final", "jar", "jar" } ) + tracker.Request(Artifact { "org.apache.ant", "ant" ,"1.9.4", "jar", "jar" } ) + tracker.Request(Artifact { "org.springframework", "spring-core", "4.1.6.RELEASE", "jar", "jar" } ) + + go func() { + time.Sleep(100) + //tracker.Request(Artifact { "commons-io", "commons-io", "2.6", "jar", "jar"}) + tracker.Request(Artifact { "commons-lang", "commons-lang", "2.4", "jar", "jar"}) + }() + + err := tracker.Wait() + if err != nil { + fmt.Printf("[ERROR] %s\n", err) + } +} + + + + + + diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..e017b1f --- /dev/null +++ b/utils.go @@ -0,0 +1,14 @@ +package main + +import ( + "strings" +) + +func ToPath(a Artifact) string { + groupPath := strings.Join(strings.Split(a.Group, "."), "/") + return groupPath + "/" + a.Id + "/" + a.Version +} + +func ToUrl(base string, a Artifact) string { + return base + "/" + ToPath(a) + "/" + a.Id + "-" + a.Version + "." + a.Ext +} diff --git a/utils_test.go b/utils_test.go new file mode 100644 index 0000000..e4ba3e1 --- /dev/null +++ b/utils_test.go @@ -0,0 +1,17 @@ +package main +import ( + "testing" +) + + +func asset(t *testing.T, exp interface{}, act interface{}, msg string) { + if exp != act { + t.Errorf(msg + " - expected: %s actual: %s", exp, act) + } +} + +func assetError(t *testing.T, err error, msg string) { + if err != nil { + t.Errorf(msg + " - Error: %s", err) + } +} \ No newline at end of file