Merge pull request #74 from GoogleCloudPlatform/release

Rename to cloud-build-local.

CONTRIBUTING.md

@@ -1,5 +1,4 @@
-Contributor License Agreements
-------------------------------
+## Contributor License Agreements
 
 **This project is not yet set up to accept external contributions.**
 
@@ -8,16 +7,16 @@ Contributor License Agreements
 Before we can accept your pull requests you'll need to sign a Contributor
 License Agreement (CLA):
 
-*  If you are an individual writing original source code and you own the
-   intellectual property, then you'll need to sign an
-   [individual CLA](https://developers.google.com/open-source/cla/individual).
+*   If you are an individual writing original source code and you own the
+    intellectual property, then you'll need to sign an
+    [individual CLA](https://developers.google.com/open-source/cla/individual).
 
-*  If you work for a company that wants to allow you to contribute your work,
-   then you'll need to sign a
-   [corporate CLA](https://developers.google.com/open-source/cla/corporate>).
+*   If you work for a company that wants to allow you to contribute your work,
+    then you'll need to sign a
+    [corporate CLA](https://developers.google.com/open-source/cla/corporate>).
 
-You can sign these electronically (just scroll to the bottom). After that,
-we'll be able to accept your pull requests.
+You can sign these electronically (just scroll to the bottom). After that, we'll
+be able to accept your pull requests.
 
 ## Developing the Local Builder
 
@@ -29,18 +28,18 @@ To build and test the Local Builder, you need a working
 Run the following commands to install the Local Builder tool:
 
 ```
-go get github.com/GoogleCloudPlatform/container-builder-local
-go install github.com/GoogleCloudPlatform/container-builder-local
+go get github.com/GoogleCloudPlatform/cloud-build-local
+go install github.com/GoogleCloudPlatform/cloud-build-local
 ```
 
 To run a build:
 
 ```
-./bin/container-builder-local --dryrun=false --config=path/to/cloudbuild.yaml path/to/code
+./bin/cloud-build-local --dryrun=false --config=path/to/cloudbuild.yaml path/to/code
 ```
 
 To run the tests for Local Builder (without the vendored libraries):
 
 ```
-go test $(go list github.com/GoogleCloudPlatform/container-builder-local/... | grep -v vendor)
+go test $(go list github.com/GoogleCloudPlatform/cloud-build-local/... | grep -v vendor)
 ```

README.md

@@ -1,26 +1,27 @@
-# Google Container Builder Local Builder
+# Google Cloud Build Local Builder
 
-**Local Builder** runs [Google Container Builder] locally,
-allowing easier debugging, execution of builds on your own hardware,
-and integration into local build and test workflows.
+**Local Builder** runs [Google Cloud Build] locally, allowing easier debugging,
+execution of builds on your own hardware, and integration into local build and
+test workflows.
 
-----
+--------------------------------------------------------------------------------
 
 ## Prerequisites
 
 1.  Ensure you have installed:
-    * [gcloud](https://cloud.google.com/sdk/docs/quickstarts)
-    * [Docker](https://www.docker.com/)
-    * [Go](https://golang.org/doc/install) (if you want to compile Local
-      Builder from source)
+
+    *   [gcloud](https://cloud.google.com/sdk/docs/quickstarts)
+    *   [Docker](https://www.docker.com/)
+    *   [Go](https://golang.org/doc/install) (if you want to compile Local
+        Builder from source)
 
 2.  If the build needs to access a private Google Container Registry, install
     and configure the
     [Docker credential helper](https://github.com/GoogleCloudPlatform/docker-credential-gcr)
     for Google Container Registry.
 
-3.  Configure your project for the gcloud tool, where `[PROJECT_ID]` is
-    your Cloud Platform project ID:
+3.  Configure your project for the gcloud tool, where `[PROJECT_ID]` is your
+    Cloud Platform project ID:
 
     ```
     gcloud config set project [PROJECT-ID]
@@ -31,49 +32,50 @@ and integration into local build and test workflows.
 1.  Install by running the following command:
 
     ```
-    gcloud components install container-builder-local
+    gcloud components install cloud-build-local
     ```
 
-    After successful installation, you will have `container-builder-local` in
-    your PATH as part of the Google Cloud SDK binaries.
+    After successful installation, you will have `cloud-build-local` in your
+    PATH as part of the Google Cloud SDK binaries.
 
 2.  To see all of the commands, run:
 
     ```
-    $ container-builder-local --help
+    $ cloud-build-local --help
     ```
 
-    The Local Builder's command is `$ container-builder-local`.
-
+    The Local Builder's command is `$ cloud-build-local`.
 
 ## Download the latest binaries
 
 The latest binaries are available in a GCS bucket.
 
-[Download](https://storage.googleapis.com/container-builder-local/container-builder-local_latest.tar.gz) the latest binaries from GCS.
+[Download](https://storage.googleapis.com/cloud-build-local/cloud-build-local_latest.tar.gz)
+the latest binaries from GCS.
 
 To run a build:
 
 ```
-./container-builder-local_{linux,darwin}_{386,amd64}-v<latest_tag> --dryrun=false --config=path/to/cloudbuild.yaml path/to/code
+./cloud-build-local_{linux,darwin}_{386,amd64}-v<latest_tag> --dryrun=false --config=path/to/cloudbuild.yaml path/to/code
 ```
 
 ## Developing and contributing to the Local Builder
 
-See the [contributing instructions](https://github.com/GoogleCloudPlatform/container-builder-local/blob/master/CONTRIBUTING.md).
+See the
+[contributing instructions](https://github.com/GoogleCloudPlatform/cloud-build-local/blob/master/CONTRIBUTING.md).
 
 ## Limitations
 
-*  Only one build can be run at a time on a given host.
-*  The tool works on the following platforms:
-   * Linux
-   * macOS
+*   Only one build can be run at a time on a given host.
+*   The tool works on the following platforms:
+    *   Linux
+    *   macOS
 
 ## Support
 
 File issues here on gitHub, email `[email protected]`, or join our
-[Slack channel] if you have general questions about Local Builder or
-Container Builder.
+[Slack channel] if you have general questions about Local Builder or Container
+Builder.
 
-[Google Container Builder]: http://cloud.google.com/container-builder/
+[Google Cloud Build]: http://cloud.google.com/cloud-build/
 [Slack channel]: https://googlecloud-community.slack.com/messages/C4KCRJL4D/details/

build/build.go

@@ -36,10 +36,11 @@ import (
 	pb "google.golang.org/genproto/googleapis/devtools/cloudbuild/v1"
 	"github.com/golang/protobuf/ptypes"
 
-	"github.com/GoogleCloudPlatform/container-builder-local/common"
-	"github.com/GoogleCloudPlatform/container-builder-local/gsutil"
-	"github.com/GoogleCloudPlatform/container-builder-local/runner"
-	"github.com/GoogleCloudPlatform/container-builder-local/volume"
+	"github.com/GoogleCloudPlatform/cloud-build-local/common"
+	"github.com/GoogleCloudPlatform/cloud-build-local/gsutil"
+	"github.com/GoogleCloudPlatform/cloud-build-local/logger"
+	"github.com/GoogleCloudPlatform/cloud-build-local/runner"
+	"github.com/GoogleCloudPlatform/cloud-build-local/volume"
 	"github.com/spf13/afero"
 	"google.golang.org/api/cloudkms/v1"
 	"golang.org/x/oauth2"
@@ -76,20 +77,6 @@ var (
 	timeNow = time.Now
 )
 
-// Logger encapsulates logging build output.
-type Logger interface {
-	WriteMainEntry(msg string)
-	Close() error
-	MakeWriter(prefix string, stepIdx int, stdout bool) io.Writer
-}
-
-// EventLogger encapsulates logging events about build steps
-// starting/finishing.
-type EventLogger interface {
-	StartStep(ctx context.Context, stepIdx int, startTime time.Time) error
-	FinishStep(ctx context.Context, stepIdx int, success bool, endTime time.Time) error
-}
-
 type imageDigest struct {
 	tag, digest string
 }
@@ -101,8 +88,7 @@ type Build struct {
 	Request          pb.Build
 	HasMultipleSteps bool
 	TokenSource      oauth2.TokenSource
-	Log              Logger
-	EventLogger      EventLogger
+	Log              logger.Logger
 	status           BuildStatus
 	imageDigests     []imageDigest // docker image tag to digest (for built images)
 	stepDigests      []string      // build step index to digest (for build steps)
@@ -183,14 +169,13 @@ type kms interface {
 
 // New constructs a new Build.
 func New(r runner.Runner, b pb.Build, ts oauth2.TokenSource,
-	bl Logger, eventLogger EventLogger, hostWorkspaceDir string, fs afero.Fs, local, push, dryrun bool) *Build {
+	bl logger.Logger, hostWorkspaceDir string, fs afero.Fs, local, push, dryrun bool) *Build {
 	return &Build{
 		Runner:           r,
 		Request:          b,
 		TokenSource:      ts,
 		stepDigests:      make([]string, len(b.Steps)),
 		Log:              bl,
-		EventLogger:      eventLogger,
 		Done:             make(chan struct{}),
 		times:            map[BuildStatus]time.Duration{},
 		lastStateStart:   timeNow(),
@@ -199,7 +184,7 @@ func New(r runner.Runner, b pb.Build, ts oauth2.TokenSource,
 		local:            local,
 		push:             push,
 		dryrun:           dryrun,
-		gsutilHelper:     gsutil.New(r, fs),
+		gsutilHelper:     gsutil.New(r, fs, bl),
 		fs:               fs,
 	}
 }
@@ -833,7 +818,7 @@ func (b *Build) getKMSClient() (kms, error) {
 	// when spoofing metadata works by IP. Until then, we'll just fetch the token
 	// and pass it to all HTTP requests.
 	svc, err := cloudkms.New(&http.Client{
-		Transport: &tokenTransport{b.TokenSource},
+		Transport: &common.TokenTransport{b.TokenSource},
 	})
 	if err != nil {
 		return nil, err
@@ -883,11 +868,6 @@ func (b *Build) timeAndRunStep(ctx context.Context, idx int, waitChans []chan st
 	b.Timing.BuildSteps[idx] = &TimeSpan{Start: when}
 	b.mu.Unlock()
 
-	if err := b.EventLogger.StartStep(ctx, idx, when); err != nil {
-		log.Printf("Error publishing start-step event: %v", err)
-
-	}
-
 	err := b.runStep(ctx, idx)
 
 	when = timeNow()
@@ -905,13 +885,6 @@ func (b *Build) timeAndRunStep(ctx context.Context, idx int, waitChans []chan st
 	}
 	b.mu.Unlock()
 
-	// We use a background context to send the FinishStep message because ctx may
-	// have been timed out or cancelled.
-	if err := b.EventLogger.FinishStep(context.Background(), idx, err == nil, when); err != nil {
-		log.Printf("Error publishing finish-step event: %v", err)
-
-	}
-
 	// If another step executing in parallel fails and sends an error, this step
 	// will be blocked from sending an error on the channel.
 	// Listen for context cancellation so that the goroutine exits.
@@ -1040,11 +1013,8 @@ func (b *Build) runStep(ctx context.Context, idx int) error {
 		}
 	}
 
-	if err := b.Runner.Run(ctx, args, nil, outWriter, errWriter, ""); err != nil {
-		return err
-	}
-
-	return nil
+	buildErr := b.Runner.Run(ctx, args, nil, outWriter, errWriter, "")
+	return buildErr
 }
 
 
@@ -1318,6 +1288,12 @@ func (b *Build) pushImages(ctx context.Context) error {
 	return nil
 }
 
+// GCS URL to bucket
+func extractGCSBucket(url string) string {
+	toks := strings.SplitN(strings.TrimPrefix(url, "gs://"), "/", 2)
+	return fmt.Sprintf("gs://%s", toks[0])
+}
+
 var newUUID = uuid.New
 
 // pushArtifacts pushes ArtifactObjects to a specified bucket.
@@ -1326,14 +1302,16 @@ func (b *Build) pushArtifacts(ctx context.Context) error {
 		return nil
 	}
 
-	// Check that the GCS bucket exists.
+	// Only verify that the GCS bucket exists.
+	// If they specify a directory path in the bucket that doesn't exist, gsutil will create it for them.
 
-	bucket := b.Request.Artifacts.Objects.Location
+	location := b.Request.Artifacts.Objects.Location
+	bucket := extractGCSBucket(location)
 	if err := b.gsutilHelper.VerifyBucket(ctx, bucket); err != nil {
 		return err
 	}
 
-	// Upload specified artifacts from the workspace to the GCS bucket.
+	// Upload specified artifacts from the workspace to the GCS location.
 	workdir := containerWorkspaceDir
 	if dir := b.Request.GetSource().GetRepoSource().GetDir(); dir != "" {
 		workdir = path.Join(workdir, dir)
@@ -1348,30 +1326,31 @@ func (b *Build) pushArtifacts(ctx context.Context) error {
 	b.Timing.ArtifactsPushes = &TimeSpan{Start: timeNow()}
 	b.mu.Unlock()
 
-	b.Log.WriteMainEntry(fmt.Sprintf("Artifacts will be uploaded to %s", bucket))
+	b.Log.WriteMainEntry(fmt.Sprintf("Artifacts will be uploaded to %s using gsutil cp", bucket))
 	results := []*pb.ArtifactResult{}
 	for _, src := range b.Request.Artifacts.Objects.Paths {
-		b.Log.WriteMainEntry(fmt.Sprintf("%s: uploading matching files...", src))
-		r, err := b.gsutilHelper.UploadArtifacts(ctx, flags, src, bucket)
+		b.Log.WriteMainEntry(fmt.Sprintf("%s: Uploading path....", src))
+		r, err := b.gsutilHelper.UploadArtifacts(ctx, flags, src, location)
 		if err != nil {
-			return fmt.Errorf("could not upload %s to %s; err = %v", src, bucket, err)
+			return fmt.Errorf("could not upload %s to %s; err = %v", src, location, err)
 		}
 
 		results = append(results, r...)
 		b.Log.WriteMainEntry(fmt.Sprintf("%s: %d matching files uploaded", src, len(r)))
 	}
 	numArtifacts := int64(len(results))
-	b.Log.WriteMainEntry(fmt.Sprintf("%d total artifacts uploaded to %s", numArtifacts, bucket))
+	b.Log.WriteMainEntry(fmt.Sprintf("%d total artifacts uploaded to %s", numArtifacts, location))
 
 	b.mu.Lock()
 	b.Timing.ArtifactsPushes.End = timeNow()
 	b.mu.Unlock()
 
-	// Write a JSON manifest for the artifacts and upload to the GCS bucket.
+	// Write a JSON manifest for the artifacts and upload to the GCS location.
 	filename := fmt.Sprintf("artifacts-%s.json", b.Request.Id)
-	artifactManifest, err := b.gsutilHelper.UploadArtifactsManifest(ctx, flags, filename, bucket, results)
+	b.Log.WriteMainEntry(fmt.Sprintf("Uploading manifest %s", filename))
+	artifactManifest, err := b.gsutilHelper.UploadArtifactsManifest(ctx, flags, filename, location, results)
 	if err != nil {
-		return fmt.Errorf("could not upload %s to %s; err = %v", filename, bucket, err)
+		return fmt.Errorf("could not upload %s to %s; err = %v", filename, location, err)
 	}
 	b.Log.WriteMainEntry(fmt.Sprintf("Artifact manifest located at %s", artifactManifest))
 
@@ -1380,21 +1359,3 @@ func (b *Build) pushArtifacts(ctx context.Context) error {
 
 	return nil
 }
-
-// tokenTransport is a RoundTripper that automatically applies OAuth
-// credentials from the token source.
-//
-// This can be replaced by google.DefaultClient when metadata spoofing works by
-// IP address (b/33233310).
-type tokenTransport struct {
-	ts oauth2.TokenSource
-}
-
-func (t *tokenTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	tok, err := t.ts.Token()
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("Authorization", "Bearer "+tok.AccessToken)
-	return http.DefaultTransport.RoundTrip(req)
-}