Add buftarget package

TODO.txt

@@ -12,15 +12,3 @@ NOTE: We are not allowing cross-workspace finding for include_package_files=true
 
 - Changed protoc-gen-buf-lint and protoc-gen-buf-breaking configuration, deprecated old configurations
 - Need to manually test protoc-gen-buf-breaking
-
-$ bothbuf build google/ads/admob/v1/admob_api.proto
-OLD
-OLD STDOUT:
-OLD EXIT CODE: 100
-OLD STDERR:
-google/ads/admob/v1/admob_api.proto:19:8:stat google/ads/admob/v1/admob_resources.proto: file does not exist
-NEW
-NEW STDOUT:
-NEW EXIT CODE: 100
-NEW STDERR:
-/Users/pedge/git/googleapis/google/ads/admob/v1/admob_api.proto:19:8:stat google/ads/admob/v1/admob_resources.proto: file does not exist

private/buf/bufctl/controller.go

@@ -792,7 +792,7 @@ func (c *controller) getWorkspaceForProtoFileRef(
 		// TODO FUTURE: Feed flag names through to here.
 		return nil, fmt.Errorf("--exclude-path is not valid for use with .proto file references")
 	}
-	readBucketCloser, err := c.buffetchReader.GetSourceReadBucketCloser(
+	readBucketCloser, bucketTargeting, err := c.buffetchReader.GetSourceReadBucketCloser(
 		ctx,
 		c.container,
 		protoFileRef,
@@ -804,19 +804,9 @@ func (c *controller) getWorkspaceForProtoFileRef(
 	defer func() {
 		retErr = multierr.Append(retErr, readBucketCloser.Close())
 	}()
-	// The ProtoFilePath is still relative to the input bucket, not the bucket
-	// retrieved from buffetch. Treat the path just as we do with targetPaths
-	// and externalPaths in withPathsForBucketExtender.
-	protoFilePath, err := readBucketCloser.PathForExternalPath(protoFileRef.ProtoFilePath())
-	if err != nil {
-		return nil, err
-	}
 	options := []bufworkspace.WorkspaceBucketOption{
-		bufworkspace.WithTargetSubDirPath(
-			readBucketCloser.SubDirPath(),
-		),
 		bufworkspace.WithProtoFileTargetPath(
-			protoFilePath,
+			protoFileRef.ProtoFilePath(),
 			protoFileRef.IncludePackageFiles(),
 		),
 		bufworkspace.WithConfigOverride(
@@ -832,6 +822,7 @@ func (c *controller) getWorkspaceForProtoFileRef(
 	return c.workspaceProvider.GetWorkspaceForBucket(
 		ctx,
 		readBucketCloser,
+		bucketTargeting,
 		options...,
 	)
 }
@@ -841,7 +832,7 @@ func (c *controller) getWorkspaceForSourceRef(
 	sourceRef buffetch.SourceRef,
 	functionOptions *functionOptions,
 ) (_ bufworkspace.Workspace, retErr error) {
-	readBucketCloser, err := c.buffetchReader.GetSourceReadBucketCloser(
+	readBucketCloser, bucketTargeting, err := c.buffetchReader.GetSourceReadBucketCloser(
 		ctx,
 		c.container,
 		sourceRef,
@@ -853,18 +844,7 @@ func (c *controller) getWorkspaceForSourceRef(
 	defer func() {
 		retErr = multierr.Append(retErr, readBucketCloser.Close())
 	}()
-	functionOptions, err = functionOptions.withPathsForBucketExtender(readBucketCloser)
-	if err != nil {
-		return nil, err
-	}
 	options := []bufworkspace.WorkspaceBucketOption{
-		bufworkspace.WithTargetSubDirPath(
-			readBucketCloser.SubDirPath(),
-		),
-		bufworkspace.WithTargetPaths(
-			functionOptions.targetPaths,
-			functionOptions.targetExcludePaths,
-		),
 		bufworkspace.WithConfigOverride(
 			functionOptions.configOverride,
 		),
@@ -878,6 +858,7 @@ func (c *controller) getWorkspaceForSourceRef(
 	return c.workspaceProvider.GetWorkspaceForBucket(
 		ctx,
 		readBucketCloser,
+		bucketTargeting,
 		options...,
 	)
 }
@@ -887,7 +868,7 @@ func (c *controller) getWorkspaceDepManagerForDirRef(
 	dirRef buffetch.DirRef,
 	functionOptions *functionOptions,
 ) (_ bufworkspace.WorkspaceDepManager, retErr error) {
-	readWriteBucket, err := c.buffetchReader.GetDirReadWriteBucket(
+	readWriteBucket, bucketTargeting, err := c.buffetchReader.GetDirReadWriteBucket(
 		ctx,
 		c.container,
 		dirRef,
@@ -901,9 +882,7 @@ func (c *controller) getWorkspaceDepManagerForDirRef(
 	return c.workspaceDepManagerProvider.GetWorkspaceDepManager(
 		ctx,
 		readWriteBucket,
-		bufworkspace.WithTargetSubDirPath(
-			readWriteBucket.SubDirPath(),
-		),
+		bucketTargeting,
 	)
 }
 

private/buf/bufctl/option.go

@@ -14,7 +14,9 @@
 
 package bufctl
 
-import "github.com/bufbuild/buf/private/buf/buffetch"
+import (
+	"github.com/bufbuild/buf/private/buf/buffetch"
+)
 
 type ControllerOption func(*controller)
 
@@ -144,28 +146,6 @@ func newFunctionOptions(controller *controller) *functionOptions {
 	}
 }
 
-func (f *functionOptions) withPathsForBucketExtender(
-	bucketExtender buffetch.BucketExtender,
-) (*functionOptions, error) {
-	deref := *f
-	c := &deref
-	for i, inputTargetPath := range c.targetPaths {
-		targetPath, err := bucketExtender.PathForExternalPath(inputTargetPath)
-		if err != nil {
-			return nil, err
-		}
-		c.targetPaths[i] = targetPath
-	}
-	for i, inputTargetExcludePath := range c.targetExcludePaths {
-		targetExcludePath, err := bucketExtender.PathForExternalPath(inputTargetExcludePath)
-		if err != nil {
-			return nil, err
-		}
-		c.targetExcludePaths[i] = targetExcludePath
-	}
-	return c, nil
-}
-
 func (f *functionOptions) getGetReadBucketCloserOptions() []buffetch.GetReadBucketCloserOption {
 	var getReadBucketCloserOptions []buffetch.GetReadBucketCloserOption
 	if f.copyToInMemory {
@@ -174,6 +154,18 @@ func (f *functionOptions) getGetReadBucketCloserOptions() []buffetch.GetReadBuck
 			buffetch.GetReadBucketCloserWithCopyToInMemory(),
 		)
 	}
+	if len(f.targetPaths) > 0 {
+		getReadBucketCloserOptions = append(
+			getReadBucketCloserOptions,
+			buffetch.GetReadBucketCloserWithTargetPaths(f.targetPaths),
+		)
+	}
+	if len(f.targetExcludePaths) > 0 {
+		getReadBucketCloserOptions = append(
+			getReadBucketCloserOptions,
+			buffetch.GetReadBucketCloserWithTargetExcludePaths(f.targetExcludePaths),
+		)
+	}
 	if f.configOverride != "" {
 		// If we have a config override, we do not search for buf.yamls or buf.work.yamls,
 		// instead acting as if the config override was the only configuration file available.

private/buf/buffetch/buffetch.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 
 	"github.com/bufbuild/buf/private/buf/buffetch/internal"
+	"github.com/bufbuild/buf/private/buf/buftarget"
 	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule"
 	"github.com/bufbuild/buf/private/pkg/app"
@@ -321,7 +322,7 @@ type SourceReader interface {
 		container app.EnvStdinContainer,
 		sourceRef SourceRef,
 		options ...GetReadBucketCloserOption,
-	) (ReadBucketCloser, error)
+	) (ReadBucketCloser, buftarget.BucketTargeting, error)
 }
 
 // GetReadBucketCloserOption is an option for a GetSourceReadBucketCloser call.
@@ -345,6 +346,22 @@ func GetReadBucketCloserWithNoSearch() GetReadBucketCloserOption {
 	}
 }
 
+// GetReadBucketCloserWithTargetPaths sets the targets paths for bucket targeting information
+// returned with the bucket.
+func GetReadBucketCloserWithTargetPaths(targetPaths []string) GetReadBucketCloserOption {
+	return func(getReadBucketCloserOptions *getReadBucketCloserOptions) {
+		getReadBucketCloserOptions.targetPaths = targetPaths
+	}
+}
+
+// GetReadBucketCloserWithTargetExcludePaths sets the target exclude paths for bucket targeting
+// information returned with the bucket.
+func GetReadBucketCloserWithTargetExcludePaths(targetExcludePaths []string) GetReadBucketCloserOption {
+	return func(getReadBucketCloserOptions *getReadBucketCloserOptions) {
+		getReadBucketCloserOptions.targetExcludePaths = targetExcludePaths
+	}
+}
+
 // DirReader is a dir reader.
 type DirReader interface {
 	// GetDirReadWriteBucket gets the dir bucket.
@@ -353,7 +370,7 @@ type DirReader interface {
 		container app.EnvStdinContainer,
 		dirRef DirRef,
 		options ...GetReadWriteBucketOption,
-	) (ReadWriteBucket, error)
+	) (ReadWriteBucket, buftarget.BucketTargeting, error)
 }
 
 // GetReadWriteBucketOption is an option for a GetDirReadWriteBucket call.
@@ -369,6 +386,22 @@ func GetReadWriteBucketWithNoSearch() GetReadWriteBucketOption {
 	}
 }
 
+// GetReadWriteBucketWithTargetPaths sets the target paths for the bucket targeting information
+// returned with the bucket.
+func GetReadWriteBucketWithTargetPaths(targetPaths []string) GetReadWriteBucketOption {
+	return func(getReadWriteBucketOptions *getReadWriteBucketOptions) {
+		getReadWriteBucketOptions.targetPaths = targetPaths
+	}
+}
+
+// GetReadWriteBucketWithTargetExcludePaths sets the target exclude paths for the bucket
+// targeting information returned with the bucket.
+func GetReadWriteBucketWithTargetExcludePaths(targetExcludePaths []string) GetReadWriteBucketOption {
+	return func(getReadWriteBucketOptions *getReadWriteBucketOptions) {
+		getReadWriteBucketOptions.targetExcludePaths = targetExcludePaths
+	}
+}
+
 // ModuleFetcher is a module fetcher.
 type ModuleFetcher interface {
 	// GetModuleKey gets the ModuleKey.
@@ -544,16 +577,20 @@ func GetInputConfigForString(
 }
 
 type getReadBucketCloserOptions struct {
-	noSearch       bool
-	copyToInMemory bool
+	noSearch           bool
+	copyToInMemory     bool
+	targetPaths        []string
+	targetExcludePaths []string
 }
 
 func newGetReadBucketCloserOptions() *getReadBucketCloserOptions {
 	return &getReadBucketCloserOptions{}
 }
 
 type getReadWriteBucketOptions struct {
-	noSearch bool
+	noSearch           bool
+	targetPaths        []string
+	targetExcludePaths []string
 }
 
 func newGetReadWriteBucketOptions() *getReadWriteBucketOptions {

private/buf/buffetch/internal/internal.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 	"strconv"
 
+	"github.com/bufbuild/buf/private/buf/buftarget"
 	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule"
 	"github.com/bufbuild/buf/private/pkg/app"
@@ -183,7 +184,7 @@ type ProtoFileRef interface {
 	BucketRef
 	// Path is the normalized path to the file reference.
 	Path() string
-	// IncludePackageFiles says to include the same package files TODO update comment
+	// IncludePackageFiles says to include the files from the same package files
 	IncludePackageFiles() bool
 	FileScheme() FileScheme
 	protoFileRef()
@@ -411,12 +412,6 @@ type BucketExtender interface {
 	// the directory that contained this terminate file, and the subDirPath will be the sub-direftory of
 	// the actual asset relative to the terminate file.
 	SubDirPath() string
-
-	// PathForExternalPath takes a path external to the asset and converts it to
-	// a path that is relative to the asset.
-	//
-	// The returned path will be normalized and validated.
-	PathForExternalPath(externalPath string) (string, error)
 }
 
 // ReadBucketCloser is a bucket returned from GetReadBucketCloser.
@@ -443,23 +438,19 @@ type Reader interface {
 		options ...GetFileOption,
 	) (io.ReadCloser, error)
 	// GetReadBucketCloser gets the bucket.
-	//
-	// If done for a ProtoFileRef, the Bucket will be for the enclosing module or workspace via
-	// terminateFileNames or protoFileTerminateFileNames. No filtering of the Bucket is performed, this is
-	// the responsibility of the caller.
 	GetReadBucketCloser(
 		ctx context.Context,
 		container app.EnvStdinContainer,
 		bucketRef BucketRef,
 		options ...GetReadBucketCloserOption,
-	) (ReadBucketCloser, error)
+	) (ReadBucketCloser, buftarget.BucketTargeting, error)
 	// GetReadWriteBucket gets the bucket.
 	GetReadWriteBucket(
 		ctx context.Context,
 		container app.EnvStdinContainer,
 		dirRef DirRef,
 		options ...GetReadWriteBucketOption,
-	) (ReadWriteBucket, error)
+	) (ReadWriteBucket, buftarget.BucketTargeting, error)
 	// GetModuleKey gets the ModuleKey.
 	GetModuleKey(
 		ctx context.Context,
@@ -829,21 +820,23 @@ func WithGetReadBucketCloserCopyToInMemory() GetReadBucketCloserOption {
 // See bufconfig.TerminateAtControllingWorkspace, which is the only thing that uses this.
 // This is used by both non-ProtoFileRefs to find the controlling workspace, AND ProtoFileRefs
 // to find the controlling workspace of an enclosing module or workspace.
-func WithGetReadBucketCloserTerminateFunc(terminateFunc TerminateFunc) GetReadBucketCloserOption {
+func WithGetReadBucketCloserTerminateFunc(terminateFunc buftarget.TerminateFunc) GetReadBucketCloserOption {
 	return func(getReadBucketCloserOptions *getReadBucketCloserOptions) {
 		getReadBucketCloserOptions.terminateFunc = terminateFunc
 	}
 }
 
-// WithGetReadBucketCloserProtoFileTerminateFunc is like WithGetReadBucketCloserTerminateFunc, but determines
-// where to stop searching for the enclosing module or workspace when given a ProtoFileRef.
-//
-// See bufconfig.TerminateAtEnclosingModuleOrWorkspaceForProtoFileRef, which is the only thing that uses this.
-// This finds the enclosing module or workspace.
-// This is only used for ProtoFileRefs.
-func WithGetReadBucketCloserProtoFileTerminateFunc(protoFileTerminateFunc TerminateFunc) GetReadBucketCloserOption {
+// WithGetReadBucketCloserTargetPaths sets the target paths for the bucket targeting information.
+func WithGetReadBucketCloserTargetPaths(targetPaths []string) GetReadBucketCloserOption {
 	return func(getReadBucketCloserOptions *getReadBucketCloserOptions) {
-		getReadBucketCloserOptions.protoFileTerminateFunc = protoFileTerminateFunc
+		getReadBucketCloserOptions.targetPaths = targetPaths
+	}
+}
+
+// WithGetReadBucketCloserTargetExcludePaths sets the target exclude paths for the bucket targeting information.
+func WithGetReadBucketCloserTargetExcludePaths(targetExcludePaths []string) GetReadBucketCloserOption {
+	return func(getReadBucketCloserOptions *getReadBucketCloserOptions) {
+		getReadBucketCloserOptions.targetExcludePaths = targetExcludePaths
 	}
 }
 
@@ -857,24 +850,24 @@ type GetReadWriteBucketOption func(*getReadWriteBucketOptions)
 // See bufconfig.TerminateAtControllingWorkspace, which is the only thing that uses this.
 // This is used by both non-ProtoFileRefs to find the controlling workspace, AND ProtoFileRefs
 // to find the controlling workspace of an enclosing module or workspace.
-func WithGetReadWriteBucketTerminateFunc(terminateFunc TerminateFunc) GetReadWriteBucketOption {
+func WithGetReadWriteBucketTerminateFunc(terminateFunc buftarget.TerminateFunc) GetReadWriteBucketOption {
 	return func(getReadWriteBucketOptions *getReadWriteBucketOptions) {
 		getReadWriteBucketOptions.terminateFunc = terminateFunc
 	}
 }
 
-// TerminateFunc is a termination function.
-//
-// This function should return true if the search should terminate at the prefix, that is
-// the prefix should be where the bucket is mapped onto.
-//
-// The original subDirPath is also given to the TerminateFunc.
-type TerminateFunc func(
-	ctx context.Context,
-	bucket storage.ReadBucket,
-	prefix string,
-	originalSubDirPath string,
-) (terminate bool, err error)
+// WithGetReadWriteBucketTargetPaths sets the target paths for the bucket targeting information.
+func WithGetReadWriteBucketTargetPaths(targetPaths []string) GetReadWriteBucketOption {
+	return func(getReadWriteBucketOptions *getReadWriteBucketOptions) {
+		getReadWriteBucketOptions.targetPaths = targetPaths
+	}
+}
+
+func WithGetReadWriteBucketTargetExcludePaths(targetExcludePaths []string) GetReadWriteBucketOption {
+	return func(getReadWriteBucketOptions *getReadWriteBucketOptions) {
+		getReadWriteBucketOptions.targetExcludePaths = targetExcludePaths
+	}
+}
 
 // PutFileOption is a PutFile option.
 type PutFileOption func(*putFileOptions)