Merge pull request #1100 from tonistiigi/print-outline

Build: Support for printing outline/targets of the current build

build/build.go

@@ -39,6 +39,7 @@ import (
 	gateway "github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/upload/uploadprovider"
+	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/apicaps"
 	"github.com/moby/buildkit/util/entitlements"
@@ -57,6 +58,10 @@ var (
 	errDockerfileConflict = errors.New("ambiguous Dockerfile source: both stdin and flag correspond to Dockerfiles")
 )
 
+const (
+	printFallbackImage = "docker/dockerfile-upstream:1.4-outline@sha256:ccd574ab34a8875c64bb6a8fb3cfae2e6d62d31b28b9f688075cc14c9b669a59"
+)
+
 type Options struct {
 	Inputs Inputs
 
@@ -81,7 +86,13 @@ type Options struct {
 	Ulimits       *opts.UlimitOpt
 
 	// Linked marks this target as exclusively linked (not requested by the user).
-	Linked bool
+	Linked    bool
+	PrintFunc *PrintFunc
+}
+
+type PrintFunc struct {
+	Name   string
+	Format string
 }
 
 type Inputs struct {
@@ -740,7 +751,7 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
 		}
 	}
 
-	if noMobyDriver != nil && !noDefaultLoad() {
+	if noMobyDriver != nil && !noDefaultLoad() && noPrintFunc(opt) {
 		var noOutputTargets []string
 		for name, opt := range opt {
 			if !opt.Linked && len(opt.Exports) == 0 {
@@ -1049,22 +1060,69 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
 						defer func() { <-done }()
 
 						cc := c
+						var printRes map[string][]byte
 						rr, err := c.Build(ctx, so, "buildx", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
-							res, err := c.Solve(ctx, req)
-							if err != nil {
-								return nil, err
-							}
-							results.Set(resultKey(dp.driverIndex, k), res)
-							if resultHandleFunc != nil {
-								resultHandleFunc(dp.driverIndex, &ResultContext{cc, res})
+							var isFallback bool
+							var origErr error
+							for {
+								if opt.PrintFunc != nil {
+									if _, ok := req.FrontendOpt["frontend.caps"]; !ok {
+										req.FrontendOpt["frontend.caps"] = "moby.buildkit.frontend.subrequests+forward"
+									} else {
+										req.FrontendOpt["frontend.caps"] += ",moby.buildkit.frontend.subrequests+forward"
+									}
+									req.FrontendOpt["requestid"] = "frontend." + opt.PrintFunc.Name
+									if isFallback {
+										req.FrontendOpt["build-arg:BUILDKIT_SYNTAX"] = printFallbackImage
+									}
+								}
+								res, err := c.Solve(ctx, req)
+								if err != nil {
+									if origErr != nil {
+										return nil, err
+									}
+									var reqErr *errdefs.UnsupportedSubrequestError
+									if !isFallback {
+										if errors.As(err, &reqErr) {
+											switch reqErr.Name {
+											case "frontend.outline", "frontend.targets":
+												isFallback = true
+												origErr = err
+												continue
+											}
+											return nil, err
+										}
+										// buildkit v0.8 vendored in Docker 20.10 does not support typed errors
+										if strings.Contains(err.Error(), "unsupported request frontend.outline") || strings.Contains(err.Error(), "unsupported request frontend.targets") {
+											isFallback = true
+											origErr = err
+											continue
+										}
+									}
+									return nil, err
+								}
+								if opt.PrintFunc != nil {
+									printRes = res.Metadata
+								}
+								results.Set(resultKey(dp.driverIndex, k), res)
+								if resultHandleFunc != nil {
+									resultHandleFunc(dp.driverIndex, &ResultContext{cc, res})
+								}
+								return res, nil
 							}
-							return res, nil
 						}, ch)
 						if err != nil {
 							return err
 						}
 						res[i] = rr
 
+						if rr.ExporterResponse == nil {
+							rr.ExporterResponse = map[string]string{}
+						}
+						for k, v := range printRes {
+							rr.ExporterResponse[k] = string(v)
+						}
+
 						d := drivers[dp.driverIndex].Driver
 						if d.IsMobyDriver() {
 							for _, e := range so.Exports {
@@ -1640,3 +1698,12 @@ func tryNodeIdentifier(configDir string) (out string) {
 	}
 	return
 }
+
+func noPrintFunc(opt map[string]Options) bool {
+	for _, v := range opt {
+		if v.PrintFunc != nil {
+			return false
+		}
+	}
+	return true
+}

commands/build.go

@@ -48,6 +48,7 @@ const defaultTargetName = "default"
 type buildOptions struct {
 	contextPath    string
 	dockerfileName string
+	printFunc      string
 
 	allow         []string
 	buildArgs     []string
@@ -122,6 +123,11 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
 		return err
 	}
 
+	printFunc, err := parsePrintFunc(in.printFunc)
+	if err != nil {
+		return err
+	}
+
 	opts := build.Options{
 		Inputs: build.Inputs{
 			ContextPath:    in.contextPath,
@@ -141,6 +147,7 @@ func runBuild(dockerCli command.Cli, in buildOptions) (err error) {
 		Tags:          in.tags,
 		Target:        in.target,
 		Ulimits:       in.ulimits,
+		PrintFunc:     printFunc,
 	}
 
 	platforms, err := platformutil.Parse(in.platforms)
@@ -307,6 +314,14 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
 
 	printWarnings(os.Stderr, printer.Warnings(), progressMode)
 
+	for k := range resp {
+		if opts[k].PrintFunc != nil {
+			if err := printResult(opts[k].PrintFunc, resp[k].ExporterResponse); err != nil {
+				return "", nil, err
+			}
+		}
+	}
+
 	return resp[defaultTargetName].ExporterResponse["containerimage.digest"], res, err
 }
 
@@ -463,6 +478,10 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 
 	flags.StringArrayVar(&options.platforms, "platform", platformsDefault, "Set target platform for build")
 
+	if isExperimental() {
+		flags.StringVar(&options.printFunc, "print", "", "Print result of information request (outline, targets)")
+	}
+
 	flags.BoolVar(&options.exportPush, "push", false, `Shorthand for "--output=type=registry"`)
 
 	flags.BoolVarP(&options.quiet, "quiet", "q", false, "Suppress the build output and print image ID on success")
@@ -481,7 +500,7 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 
 	flags.Var(options.ulimits, "ulimit", "Ulimit options")
 
-	if os.Getenv("BUILDX_EXPERIMENTAL") == "1" {
+	if isExperimental() {
 		flags.StringVar(&options.invoke, "invoke", "", "Invoke a command after the build. BUILDX_EXPERIMENTAL=1 is required.")
 	}
 
@@ -596,6 +615,34 @@ func parseContextNames(values []string) (map[string]build.NamedContext, error) {
 	return result, nil
 }
 
+func parsePrintFunc(str string) (*build.PrintFunc, error) {
+	if str == "" {
+		return nil, nil
+	}
+	csvReader := csv.NewReader(strings.NewReader(str))
+	fields, err := csvReader.Read()
+	if err != nil {
+		return nil, err
+	}
+	f := &build.PrintFunc{}
+	for _, field := range fields {
+		parts := strings.SplitN(field, "=", 2)
+		if len(parts) == 2 {
+			if parts[0] == "format" {
+				f.Format = parts[1]
+			} else {
+				return nil, errors.Errorf("invalid print field: %s", field)
+			}
+		} else {
+			if f.Name != "" {
+				return nil, errors.Errorf("invalid print value: %s", str)
+			}
+			f.Name = field
+		}
+	}
+	return f, nil
+}
+
 func writeMetadataFile(filename string, dt interface{}) error {
 	b, err := json.MarshalIndent(dt, "", "  ")
 	if err != nil {
@@ -652,3 +699,11 @@ func (w *wrapped) Error() string {
 func (w *wrapped) Unwrap() error {
 	return w.err
 }
+
+func isExperimental() bool {
+	if v, ok := os.LookupEnv("BUILDKIT_EXPERIMENTAL"); ok {
+		vv, _ := strconv.ParseBool(v)
+		return vv
+	}
+	return false
+}

commands/print.go

@@ -0,0 +1,48 @@
+package commands
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/docker/buildx/build"
+	"github.com/docker/docker/api/types/versions"
+	"github.com/moby/buildkit/frontend/subrequests"
+	"github.com/moby/buildkit/frontend/subrequests/outline"
+	"github.com/moby/buildkit/frontend/subrequests/targets"
+)
+
+func printResult(f *build.PrintFunc, res map[string]string) error {
+	switch f.Name {
+	case "outline":
+		return printValue(outline.PrintOutline, outline.SubrequestsOutlineDefinition.Version, f.Format, res)
+	case "targets":
+		return printValue(targets.PrintTargets, targets.SubrequestsTargetsDefinition.Version, f.Format, res)
+	case "subrequests.describe":
+		return printValue(subrequests.PrintDescribe, subrequests.SubrequestsDescribeDefinition.Version, f.Format, res)
+	default:
+		if dt, ok := res["result.txt"]; ok {
+			fmt.Print(dt)
+		} else {
+			log.Printf("%s %+v", f, res)
+		}
+	}
+	return nil
+}
+
+type printFunc func([]byte, io.Writer) error
+
+func printValue(printer printFunc, version string, format string, res map[string]string) error {
+	if format == "json" {
+		fmt.Fprintln(os.Stdout, res["result.json"])
+		return nil
+	}
+
+	if res["version"] != "" && versions.LessThan(version, res["version"]) && res["result.txt"] != "" {
+		// structure is too new and we don't know how to print it
+		fmt.Fprint(os.Stdout, res["result.txt"])
+		return nil
+	}
+	return printer([]byte(res["result.json"]), os.Stdout)
+}

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/hashicorp/go-cty-funcs v0.0.0-20200930094925-2721b1e36840
 	github.com/hashicorp/hcl/v2 v2.8.2
-	github.com/moby/buildkit v0.10.1-0.20220721175135-c75998aec3d4
+	github.com/moby/buildkit v0.10.1-0.20220809151411-8488654e899b
 	github.com/morikuni/aec v1.0.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799
@@ -31,7 +31,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.4.1
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
-	google.golang.org/grpc v1.45.0
+	google.golang.org/grpc v1.47.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.23.5
 	k8s.io/apimachinery v0.23.5
@@ -95,7 +95,7 @@ require (
 	github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
-	github.com/klauspost/compress v1.15.1 // indirect
+	github.com/klauspost/compress v1.15.7 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
@@ -110,7 +110,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/opencontainers/runc v1.1.3 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.12.1 // indirect
+	github.com/prometheus/client_golang v1.12.2 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
@@ -138,9 +138,9 @@ require (
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
 	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
+	google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect