Init support for remote buf plugins

go.mod

@@ -7,8 +7,8 @@ toolchain go1.23.2
 require (
 	buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.34.2-20240928190436-5e8abcfd7a7e.2
 	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.35.1-20240920164238-5a7b106cbb87.1
-	buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20241007185750-f2282fe532fe.1
-	buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.1-20241007185750-f2282fe532fe.1
+	buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-00000000000000-71a5dade3a37.1
+	buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.1-00000000000000-71a5dade3a37.1
 	buf.build/go/bufplugin v0.5.0
 	buf.build/go/protoyaml v0.2.0
 	buf.build/go/spdx v0.2.0

go.sum

@@ -2,10 +2,10 @@ buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.34.2-20240928190436-5e
 buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.34.2-20240928190436-5e8abcfd7a7e.2/go.mod h1:B+9TKHRYqoAUW57pLjhkLOnBCu0DQYMV+f7imQ9nXwI=
 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.35.1-20240920164238-5a7b106cbb87.1 h1:9wP6ZZYWnF2Z0TxmII7m3XNykxnP4/w8oXeth6ekcRI=
 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.35.1-20240920164238-5a7b106cbb87.1/go.mod h1:Duw/9JoXkXIydyASnLYIiufkzySThoqavOsF+IihqvM=
-buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20241007185750-f2282fe532fe.1 h1:tVBsEQVsOz3Xsoy90GOefAgn6siKtfFZxsst39/hJNU=
-buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20241007185750-f2282fe532fe.1/go.mod h1:of6bEOQJUQbBMVsPiLQiyD+yPRze0FClQv2u1CmgDdQ=
-buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.1-20241007185750-f2282fe532fe.1 h1:jvB3uNDZnP89xIFy8WHeYgCz86vv7mWRyv2Wgp+8hoI=
-buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.1-20241007185750-f2282fe532fe.1/go.mod h1:wDNiJdkMLOTfEL3o1lBgh+9EVhwGORcz+Aoq6hGk33Y=
+buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-00000000000000-71a5dade3a37.1 h1:/MdNdhGvUXIzanwcfoTB8RjI76j2OHzry5vpWhotSgw=
+buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-00000000000000-71a5dade3a37.1/go.mod h1:3JDqdhwZaft99WviYeeojgQfEa5ewN6lHiNmB4Fovvg=
+buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.1-00000000000000-71a5dade3a37.1 h1:IIWdVuv5fEI54rfZXlyN+iuQlqJPVRvcyVbaE2oXsgs=
+buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.1-00000000000000-71a5dade3a37.1/go.mod h1:wDNiJdkMLOTfEL3o1lBgh+9EVhwGORcz+Aoq6hGk33Y=
 buf.build/gen/go/pluginrpc/pluginrpc/protocolbuffers/go v1.34.2-20241007202033-cf42259fcbfc.2 h1:QPJpr4Jtj5AN+LXVz+r+pT8dvPkznkRPotxYSN4zXYE=
 buf.build/gen/go/pluginrpc/pluginrpc/protocolbuffers/go v1.34.2-20241007202033-cf42259fcbfc.2/go.mod h1:GjH0gjlY/ns16X8d6eaXV2W+6IFwsO5Ly9WVnzyd1E0=
 buf.build/go/bufplugin v0.5.0 h1:pmK1AloAMp+4woH5hEisK9qVmDdLySzIKexUUVZLJ2Q=

private/buf/bufcli/cache.go

@@ -27,6 +27,10 @@ import (
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulecache"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulestore"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufplugincache"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginstore"
 	"github.com/bufbuild/buf/private/pkg/app/appext"
 	"github.com/bufbuild/buf/private/pkg/command"
 	"github.com/bufbuild/buf/private/pkg/filelock"
@@ -103,6 +107,10 @@ var (
 	//
 	// Normalized.
 	v3CacheModuleLockRelDirPath = normalpath.Join("v3", "modulelocks")
+	// v3CachePluginRelDirPath is the relative path to the files cache directory in its newest iteration.
+	//
+	// Normalized.
+	v3CachePluginRelDirPath = normalpath.Join("v3", "plugins")
 	// v3CacheWasmRuntimeRelDirPath is the relative path to the Wasm runtime cache directory in its newest iteration.
 	// This directory is used to store the Wasm runtime cache. This is an implementation specific cache and opaque outside of the runtime.
 	//
@@ -140,6 +148,21 @@ func NewCommitProvider(container appext.Container) (bufmodule.CommitProvider, er
 	)
 }
 
+// NewPluginDataProvider returns a new PluginDataProvider while creating the
+// required cache directories.
+func NewPluginDataProvider(container appext.Container) (bufplugin.PluginDataProvider, error) {
+	clientConfig, err := NewConnectClientConfig(container)
+	if err != nil {
+		return nil, err
+	}
+	return newPluginDataProvider(
+		container,
+		bufapi.NewClientProvider(
+			clientConfig,
+		),
+	)
+}
+
 // CreateWasmRuntimeCacheDir creates the cache directory for the Wasm runtime.
 //
 // This is used by the Wasm runtime to cache compiled Wasm plugins. This is an
@@ -209,6 +232,33 @@ func newModuleDataProvider(
 	), nil
 }
 
+func newPluginDataProvider(
+	container appext.Container,
+	clientProvider bufapi.ClientProvider,
+) (bufplugin.PluginDataProvider, error) {
+	if err := createCacheDir(container.CacheDirPath(), v3CachePluginRelDirPath); err != nil {
+		return nil, err
+	}
+	fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CachePluginRelDirPath)
+	storageosProvider := storageos.NewProvider() // No symlinks.
+	cacheBucket, err := storageosProvider.NewReadWriteBucket(fullCacheDirPath)
+	if err != nil {
+		return nil, err
+	}
+	delegateModuleDataProvider := bufpluginapi.NewPluingDataProvider(
+		container.Logger(),
+		clientProvider,
+	)
+	return bufplugincache.NewPluginDataProvider(
+		container.Logger(),
+		delegateModuleDataProvider,
+		bufpluginstore.NewPluginDataStore(
+			container.Logger(),
+			cacheBucket,
+		),
+	), nil
+}
+
 func newCommitProvider(
 	container appext.Container,
 	clientProvider bufapi.ClientProvider,

private/buf/bufcli/controller.go

@@ -18,6 +18,7 @@ import (
 	"github.com/bufbuild/buf/private/buf/bufctl"
 	"github.com/bufbuild/buf/private/bufpkg/bufapi"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
 	"github.com/bufbuild/buf/private/pkg/app/appext"
 )
 
@@ -49,13 +50,19 @@ func NewController(
 	if err != nil {
 		return nil, err
 	}
+	pluginDataProvider, err := newPluginDataProvider(container, clientProvider)
+	if err != nil {
+		return nil, err
+	}
 	return bufctl.NewController(
 		container.Logger(),
 		container,
 		newGraphProvider(container, clientProvider),
 		bufmoduleapi.NewModuleKeyProvider(container.Logger(), clientProvider),
 		moduleDataProvider,
 		commitProvider,
+		bufpluginapi.NewPluginKeyProvider(container.Logger(), clientProvider),
+		pluginDataProvider,
 		wktStore,
 		// TODO FUTURE: Delete defaultHTTPClient and use the one from newConfig
 		defaultHTTPClient,

private/buf/bufcli/errors.go

@@ -54,6 +54,12 @@ func NewLabelNameAlreadyExistsError(name string) error {
 	return fmt.Errorf("a label named %q already exists", name)
 }
 
+// NewPluginNameAlreadyExistsError informs the user that a plugin
+// with that name already exists.
+func NewPluginNameAlreadyExistsError(name string) error {
+	return fmt.Errorf("a plugin named %q already exists", name)
+}
+
 // NewOrganizationNotFoundError informs the user that an organization with
 // that name does not exist.
 func NewOrganizationNotFoundError(name string) error {
@@ -88,6 +94,12 @@ func NewTokenNotFoundError(tokenID string) error {
 	return fmt.Errorf("a token with ID %q does not exist", tokenID)
 }
 
+// NewPluginNotFoundError informs the user that a plugin with
+// that name does not exist.
+func NewPluginNotFoundError(name string) error {
+	return fmt.Errorf("a plugin named %q does not exist", name)
+}
+
 // NewInvalidRemoteError informs the user that the given remote is invalid.
 func NewInvalidRemoteError(err error, remote string, moduleFullName string) error {
 	var connectErr *connect.Error

private/buf/bufcli/flags_args.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	modulev1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/module/v1"
+	pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1"
 	"github.com/bufbuild/buf/private/buf/buffetch"
 	"github.com/bufbuild/buf/private/pkg/app"
 	"github.com/bufbuild/buf/private/pkg/app/appcmd"
@@ -302,6 +303,21 @@ func VisibilityFlagToVisibilityAllowUnspecified(visibility string) (modulev1.Mod
 	}
 }
 
+// VisibilityFlagToPluginVisibilityAllowUnspecified parses the given string as a pluginv1.PluginVisibility
+// where an empty string will be parsed as unspecified.
+func VisibilityFlagToPluginVisibilityAllowUnspecified(visibility string) (pluginv1beta1.PluginVisibility, error) {
+	switch visibility {
+	case publicVisibility:
+		return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_PUBLIC, nil
+	case privateVisibility:
+		return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_PRIVATE, nil
+	case "":
+		return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_UNSPECIFIED, nil
+	default:
+		return 0, fmt.Errorf("invalid visibility: %s", visibility)
+	}
+}
+
 // ArchiveStatusFlagToArchiveStatusFilter parses the given string as a modulev1.ListLabelsRequest_ArchiveFilter.
 func ArchiveStatusFlagToArchiveStatusFilter(archiveStatus string) (modulev1.ListLabelsRequest_ArchiveFilter, error) {
 	switch archiveStatus {

private/buf/bufcli/plugin_key_provider.go

@@ -0,0 +1,36 @@
+// Copyright 2020-2024 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bufcli
+
+import (
+	"github.com/bufbuild/buf/private/bufpkg/bufapi"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
+	"github.com/bufbuild/buf/private/pkg/app/appext"
+)
+
+// NewPluginKeyProvider returns a new PluginKeyProvider.
+func NewPluginKeyProvider(container appext.Container) (bufplugin.PluginKeyProvider, error) {
+	clientConfig, err := NewConnectClientConfig(container)
+	if err != nil {
+		return nil, err
+	}
+	return bufpluginapi.NewPluginKeyProvider(
+		container.Logger(),
+		bufapi.NewClientProvider(
+			clientConfig,
+		),
+	), nil
+}

private/buf/bufctl/controller.go

@@ -33,6 +33,7 @@ import (
 	"github.com/bufbuild/buf/private/bufpkg/bufimage"
 	"github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin"
 	"github.com/bufbuild/buf/private/bufpkg/bufreflect"
 	"github.com/bufbuild/buf/private/gen/data/datawkt"
 	imagev1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/image/v1"
@@ -59,6 +60,8 @@ type ImageWithConfig interface {
 	LintConfig() bufconfig.LintConfig
 	BreakingConfig() bufconfig.BreakingConfig
 	PluginConfigs() []bufconfig.PluginConfig
+	PluginKeyProvider() bufplugin.PluginKeyProvider
+	PluginDataProvider() bufplugin.PluginDataProvider
 
 	isImageWithConfig()
 }
@@ -137,6 +140,8 @@ func NewController(
 	moduleKeyProvider bufmodule.ModuleKeyProvider,
 	moduleDataProvider bufmodule.ModuleDataProvider,
 	commitProvider bufmodule.CommitProvider,
+	pluginKeyProvider bufplugin.PluginKeyProvider,
+	pluginDataProvider bufplugin.PluginDataProvider,
 	wktStore bufwktstore.Store,
 	httpClient *http.Client,
 	httpauthAuthenticator httpauth.Authenticator,
@@ -150,6 +155,8 @@ func NewController(
 		moduleKeyProvider,
 		moduleDataProvider,
 		commitProvider,
+		pluginKeyProvider,
+		pluginDataProvider,
 		wktStore,
 		httpClient,
 		httpauthAuthenticator,
@@ -171,6 +178,8 @@ type controller struct {
 	moduleDataProvider bufmodule.ModuleDataProvider
 	graphProvider      bufmodule.GraphProvider
 	commitProvider     bufmodule.CommitProvider
+	pluginKeyProvider  bufplugin.PluginKeyProvider
+	pluginDataProvider bufplugin.PluginDataProvider
 	wktStore           bufwktstore.Store
 
 	disableSymlinks           bool
@@ -194,6 +203,8 @@ func newController(
 	moduleKeyProvider bufmodule.ModuleKeyProvider,
 	moduleDataProvider bufmodule.ModuleDataProvider,
 	commitProvider bufmodule.CommitProvider,
+	pluginKeyProvider bufplugin.PluginKeyProvider,
+	pluginDataProvider bufplugin.PluginDataProvider,
 	wktStore bufwktstore.Store,
 	httpClient *http.Client,
 	httpauthAuthenticator httpauth.Authenticator,
@@ -206,6 +217,8 @@ func newController(
 		graphProvider:      graphProvider,
 		moduleDataProvider: moduleDataProvider,
 		commitProvider:     commitProvider,
+		pluginKeyProvider:  pluginKeyProvider,
+		pluginDataProvider: pluginDataProvider,
 		wktStore:           wktStore,
 	}
 	for _, option := range options {
@@ -414,6 +427,8 @@ func (c *controller) GetTargetImageWithConfigs(
 				lintConfig,
 				breakingConfig,
 				pluginConfigs,
+				c.pluginKeyProvider,
+				c.pluginDataProvider,
 			),
 		}, nil
 	default:
@@ -1064,6 +1079,9 @@ func (c *controller) buildTargetImageWithConfigs(
 				workspace.GetLintConfigForOpaqueID(module.OpaqueID()),
 				workspace.GetBreakingConfigForOpaqueID(module.OpaqueID()),
 				workspace.PluginConfigs(),
+				// TODO(ed): plugin key provider and data provider should come from the workspace.
+				c.pluginKeyProvider, // KeyProvider should read directly from the lockfile.
+				c.pluginDataProvider,
 			),
 		)
 	}

private/buf/bufctl/image_with_config.go

@@ -17,27 +17,34 @@ package bufctl
 import (
 	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
 	"github.com/bufbuild/buf/private/bufpkg/bufimage"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin"
 )
 
 type imageWithConfig struct {
 	bufimage.Image
 
-	lintConfig     bufconfig.LintConfig
-	breakingConfig bufconfig.BreakingConfig
-	pluginConfigs  []bufconfig.PluginConfig
+	lintConfig         bufconfig.LintConfig
+	breakingConfig     bufconfig.BreakingConfig
+	pluginConfigs      []bufconfig.PluginConfig
+	pluginKeyProvider  bufplugin.PluginKeyProvider
+	pluginDataProvider bufplugin.PluginDataProvider
 }
 
 func newImageWithConfig(
 	image bufimage.Image,
 	lintConfig bufconfig.LintConfig,
 	breakingConfig bufconfig.BreakingConfig,
 	pluginConfigs []bufconfig.PluginConfig,
+	pluginKeyProvider bufplugin.PluginKeyProvider,
+	pluginDataProvider bufplugin.PluginDataProvider,
 ) *imageWithConfig {
 	return &imageWithConfig{
-		Image:          image,
-		lintConfig:     lintConfig,
-		breakingConfig: breakingConfig,
-		pluginConfigs:  pluginConfigs,
+		Image:              image,
+		lintConfig:         lintConfig,
+		breakingConfig:     breakingConfig,
+		pluginConfigs:      pluginConfigs,
+		pluginKeyProvider:  pluginKeyProvider,
+		pluginDataProvider: pluginDataProvider,
 	}
 }
 
@@ -53,4 +60,12 @@ func (i *imageWithConfig) PluginConfigs() []bufconfig.PluginConfig {
 	return i.pluginConfigs
 }
 
+func (i *imageWithConfig) PluginKeyProvider() bufplugin.PluginKeyProvider {
+	return i.pluginKeyProvider
+}
+
+func (i *imageWithConfig) PluginDataProvider() bufplugin.PluginDataProvider {
+	return i.pluginDataProvider
+}
+
 func (*imageWithConfig) isImageWithConfig() {}

private/buf/bufmigrate/migrator.go

@@ -28,6 +28,7 @@ import (
 	"github.com/bufbuild/buf/private/bufpkg/bufcheck"
 	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin"
 	"github.com/bufbuild/buf/private/pkg/command"
 	"github.com/bufbuild/buf/private/pkg/normalpath"
 	"github.com/bufbuild/buf/private/pkg/slicesext"
@@ -406,6 +407,7 @@ func (m *migrator) buildBufYAMLAndBufLockFiles(
 			bufLock, err = bufconfig.NewBufLockFile(
 				bufconfig.FileVersionV2,
 				resolvedLockEntries,
+				nil,
 			)
 			if err != nil {
 				return nil, nil, err
@@ -450,6 +452,7 @@ func (m *migrator) buildBufYAMLAndBufLockFiles(
 		bufLock, err = bufconfig.NewBufLockFile(
 			bufconfig.FileVersionV2,
 			resolvedDepModuleKeys,
+			nil, // Plugins not supported in v1.
 		)
 		if err != nil {
 			return nil, nil, err
@@ -704,7 +707,7 @@ func equivalentCheckConfigInV2(
 ) (bufconfig.CheckConfig, error) {
 	// No need for custom lint/breaking plugins since there's no plugins to migrate from <=v1.
 	// TODO: If we ever need v3, then we will have to deal with this.
-	client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(runner, wasm.UnimplementedRuntime))
+	client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(runner, wasm.UnimplementedRuntime, bufplugin.NopPluginKeyProvider, bufplugin.NopPluginDataProvider))
 	if err != nil {
 		return nil, err
 	}