From fd2778471737c28bb34d6544d20ec1b58d00ad01 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <filippo@cloudflare.com>
Date: Sun, 17 Jan 2016 19:46:37 -0800
Subject: [PATCH 1/2] gb-vendor: delete old dependency before copypath during
 restore

---
 cmd/gb-vendor/restore.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cmd/gb-vendor/restore.go b/cmd/gb-vendor/restore.go
index a2eb5a0..263ad39 100644
--- a/cmd/gb-vendor/restore.go
+++ b/cmd/gb-vendor/restore.go
@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"os"
 	"path/filepath"
 
 	"github.com/constabulary/gb"
@@ -51,6 +52,12 @@ func restore(ctx *gb.Context) error {
 		dst := filepath.Join(ctx.Projectdir(), "vendor", "src", dep.Importpath)
 		src := filepath.Join(wc.Dir(), dep.Path)
 
+		if _, err := os.Stat(dst); err == nil {
+			if err := fileutils.RemoveAll(dst); err != nil {
+				return fmt.Errorf("dependency could not be deleted: %v", err)
+			}
+		}
+
 		if err := fileutils.Copypath(dst, src); err != nil {
 			return err
 		}

From e56b63d7e7d44dfabb3c90e62ed9f6c48b3385ee Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <filippo@cloudflare.com>
Date: Sun, 17 Jan 2016 19:56:40 -0800
Subject: [PATCH 2/2] gb-vendor: parallelize dependencies download in rebuild

Upstreaming FiloSottile/gvt#10 by @petrkotek
---
 cmd/gb-vendor/restore.go | 95 +++++++++++++++++++++++++++++-----------
 1 file changed, 69 insertions(+), 26 deletions(-)

diff --git a/cmd/gb-vendor/restore.go b/cmd/gb-vendor/restore.go
index 263ad39..515ea82 100644
--- a/cmd/gb-vendor/restore.go
+++ b/cmd/gb-vendor/restore.go
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
+	"sync/atomic"
 
 	"github.com/constabulary/gb"
 	"github.com/constabulary/gb/cmd"
@@ -12,23 +14,36 @@ import (
 	"github.com/constabulary/gb/vendor"
 )
 
+var (
+	rbInsecure    bool // Allow the use of insecure protocols
+	rbConnections uint // Count of concurrent download connections
+)
+
 func addRestoreFlags(fs *flag.FlagSet) {
-	fs.BoolVar(&insecure, "precaire", false, "allow the use of insecure protocols")
+	fs.BoolVar(&rbInsecure, "precaire", false, "allow the use of insecure protocols")
+	fs.UintVar(&rbConnections, "connections", 8, "count of parallel download connections")
 }
 
 var cmdRestore = &cmd.Command{
 	Name:      "restore",
-	UsageLine: "restore [-precaire]",
-	Short:     "restore dependencies from the manifest",
-	Long: `Restore vendor dependecies.
+	UsageLine: "restore [-precaire] [-connections N]",
+	Short:     "restore dependencies from manifest",
+	Long: `restore fetches the dependencies listed in the manifest.
 
 Flags:
 	-precaire
 		allow the use of insecure protocols.
+	-connections
+		count of parallel download connections.
 
 `,
 	Run: func(ctx *gb.Context, args []string) error {
-		return restore(ctx)
+		switch len(args) {
+		case 0:
+			return restore(ctx)
+		default:
+			return fmt.Errorf("restore takes no arguments")
+		}
 	},
 	AddFlags: addRestoreFlags,
 }
@@ -39,33 +54,61 @@ func restore(ctx *gb.Context) error {
 		return fmt.Errorf("could not load manifest: %v", err)
 	}
 
+	var errors uint32
+	var wg sync.WaitGroup
+	depC := make(chan vendor.Dependency)
+	for i := 0; i < int(rbConnections); i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for d := range depC {
+				if err := downloadDependency(ctx, d); err != nil {
+					fmt.Printf("%s: %v", d.Importpath, err)
+					atomic.AddUint32(&errors, 1)
+				}
+			}
+		}()
+	}
+
 	for _, dep := range m.Dependencies {
-		fmt.Printf("Getting %s\n", dep.Importpath)
-		repo, _, err := vendor.DeduceRemoteRepo(dep.Importpath, insecure)
-		if err != nil {
-			return fmt.Errorf("Could not process dependency: %s", err)
-		}
-		wc, err := repo.Checkout("", "", dep.Revision)
-		if err != nil {
-			return fmt.Errorf("Could not retrieve dependency: %s", err)
-		}
-		dst := filepath.Join(ctx.Projectdir(), "vendor", "src", dep.Importpath)
-		src := filepath.Join(wc.Dir(), dep.Path)
+		depC <- dep
+	}
+	close(depC)
+	wg.Wait()
 
-		if _, err := os.Stat(dst); err == nil {
-			if err := fileutils.RemoveAll(dst); err != nil {
-				return fmt.Errorf("dependency could not be deleted: %v", err)
-			}
-		}
+	if errors > 0 {
+		return fmt.Errorf("failed to fetch %d dependencies", errors)
+	}
 
-		if err := fileutils.Copypath(dst, src); err != nil {
-			return err
-		}
+	return nil
+}
+
+func downloadDependency(ctx *gb.Context, dep vendor.Dependency) error {
+	fmt.Printf("Getting %s\n", dep.Importpath)
+	repo, _, err := vendor.DeduceRemoteRepo(dep.Importpath, rbInsecure)
+	if err != nil {
+		return fmt.Errorf("dependency could not be processed: %s", err)
+	}
+	wc, err := repo.Checkout("", "", dep.Revision)
+	if err != nil {
+		return fmt.Errorf("dependency could not be fetched: %s", err)
+	}
+	dst := filepath.Join(ctx.Projectdir(), "vendor", "src", dep.Importpath)
+	src := filepath.Join(wc.Dir(), dep.Path)
 
-		if err := wc.Destroy(); err != nil {
-			return err
+	if _, err := os.Stat(dst); err == nil {
+		if err := fileutils.RemoveAll(dst); err != nil {
+			return fmt.Errorf("dependency could not be deleted: %v", err)
 		}
+	}
 
+	if err := fileutils.Copypath(dst, src); err != nil {
+		return err
 	}
+
+	if err := wc.Destroy(); err != nil {
+		return err
+	}
+
 	return nil
 }