From a045268fe9e0feba96fe399145b11c18779a691e Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sun, 2 Feb 2025 16:03:26 +0100 Subject: [PATCH 01/61] Update actions/upload-artifact to v4. Signed-off-by: Thomas Hallgren --- .github/actions/upload-logs/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/upload-logs/action.yaml b/.github/actions/upload-logs/action.yaml index e172c39ffa..f3e4bf44ad 100644 --- a/.github/actions/upload-logs/action.yaml +++ b/.github/actions/upload-logs/action.yaml @@ -19,7 +19,7 @@ runs: shell: bash name: Gather logs - name: Upload logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: # If an environment variable LOG_SUFFIX is set, it will be appended to the log filename. name: ${{github.job}}-logs-${{ env.LOG_SUFFIX }} From 06125d8ca2d6b52814bfdb67bbc6914eba06d95b Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sun, 2 Feb 2025 07:33:50 +0100 Subject: [PATCH 02/61] Fix panic when allowing conflicting subnets on macOS. This commit fixes a regression introduced in 2.21.0, causing the macOS client to panic when the `--allow-conflicting-subnets` connect flag was used, or when `routing.allowConflictingSubnets` was added to the client configuration. Closes #3784 Signed-off-by: Thomas Hallgren --- CHANGELOG.yml | 8 ++++++++ docs/release-notes.md | 7 +++++++ docs/release-notes.mdx | 5 +++++ pkg/vif/device.go | 2 +- pkg/vif/device_darwin.go | 37 ++++++++++++------------------------- pkg/vif/device_linux.go | 4 ---- pkg/vif/device_windows.go | 4 ---- 7 files changed, 33 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.yml b/CHANGELOG.yml index 139b920b34..0a4006be64 100644 --- a/CHANGELOG.yml +++ b/CHANGELOG.yml @@ -33,6 +33,14 @@ docDescription: >- environments, access to instantaneous feedback loops, and highly customizable development environments. items: + - version: 2.21.3 + date: (TBD) + notes: + - type: bugfix + title: Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS. + body: >- + A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the + TUN-device on macOS based clients. - version: 2.21.2 date: 2025-01-26 notes: diff --git a/docs/release-notes.md b/docs/release-notes.md index a46747fc49..0d1960bd69 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,6 +1,13 @@ [comment]: # (Code generated by relnotesgen. DO NOT EDIT.) # Telepresence Release Notes +## Version 2.21.3 +##
bugfix
Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS.
+
+ +A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the TUN-device on macOS based clients. +
+ ## Version 2.21.2 (January 26) ##
bugfix
Fix panic when agentpf.client creates a Tunnel
diff --git a/docs/release-notes.mdx b/docs/release-notes.mdx index a1c29c82e8..2690b298c0 100644 --- a/docs/release-notes.mdx +++ b/docs/release-notes.mdx @@ -7,6 +7,11 @@ import { Note, Title, Body } from '@site/src/components/ReleaseNotes' [comment]: # (Code generated by relnotesgen. DO NOT EDIT.) # Telepresence Release Notes +## Version 2.21.3 + + Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS. + A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the TUN-device on macOS based clients. + ## Version 2.21.2 (January 26) Fix panic when agentpf.client creates a Tunnel diff --git a/pkg/vif/device.go b/pkg/vif/device.go index 87a7898555..979d46a54d 100644 --- a/pkg/vif/device.go +++ b/pkg/vif/device.go @@ -33,7 +33,7 @@ func (d *device) AddSubnet(ctx context.Context, subnet netip.Prefix) (err error) // Index returns the index of this device. func (d *device) Index() uint32 { - return d.index() + return d.interfaceIndex } // Name returns the name of this device, e.g. "tun0". diff --git a/pkg/vif/device_darwin.go b/pkg/vif/device_darwin.go index 4e62248005..30ea61dc36 100644 --- a/pkg/vif/device_darwin.go +++ b/pkg/vif/device_darwin.go @@ -29,11 +29,12 @@ const ( type device struct { *channel.Endpoint - file *os.File - ctx context.Context - name string - wb bytes.Buffer - wg sync.WaitGroup + file *os.File + ctx context.Context + name string + interfaceIndex uint32 + wb bytes.Buffer + wg sync.WaitGroup } func openTun(ctx context.Context) (*device, error) { @@ -66,29 +67,19 @@ func openTun(ctx context.Context) (*device, error) { if err != nil { return nil, err } - mtu, err := getMTU(name) + iface, err := net.InterfaceByName(name) if err != nil { return nil, err } return &device{ - file: os.NewFile(uintptr(fd), ""), - ctx: ctx, - name: name, - Endpoint: channel.New(defaultDevOutQueueLen, mtu, ""), + file: os.NewFile(uintptr(fd), ""), + ctx: ctx, + name: name, + interfaceIndex: uint32(iface.Index), + Endpoint: channel.New(defaultDevOutQueueLen, uint32(iface.MTU), ""), }, nil } -func getMTU(name string) (mtu uint32, err error) { - err = withSocket(unix.AF_INET, func(fd int) error { - ifr, err := unix.IoctlGetIfreqMTU(fd, name) - if err == nil { - mtu = uint32(ifr.MTU) - } - return err - }) - return mtu, err -} - // Close closes both the tun-device and the Endpoint. This function overrides the LinkEndpoint.Close so // it can not return an error. func (d *device) Close() { @@ -106,10 +97,6 @@ func (d *device) addSubnet(_ context.Context, subnet netip.Prefix) error { return routing.Add(1, subnet, dest) } -func (d *device) index() uint32 { - panic("not implemented") -} - func (d *device) removeSubnet(_ context.Context, subnet netip.Prefix) error { to := subnet.Addr().AsSlice() to[len(to)-1] = 1 diff --git a/pkg/vif/device_linux.go b/pkg/vif/device_linux.go index e5786f22ec..ddf6ad4262 100644 --- a/pkg/vif/device_linux.go +++ b/pkg/vif/device_linux.go @@ -98,10 +98,6 @@ func (d *device) removeSubnet(ctx context.Context, pfx netip.Prefix) error { return netlink.AddrDel(link, addr) } -func (d *device) index() uint32 { - return d.interfaceIndex -} - func (d *device) getMTU() (mtu uint32, err error) { err = withSocket(unix.AF_INET, func(fd int) error { ifr, err := unix.NewIfreq(d.name) diff --git a/pkg/vif/device_windows.go b/pkg/vif/device_windows.go index bb71181a56..99885f56dd 100644 --- a/pkg/vif/device_windows.go +++ b/pkg/vif/device_windows.go @@ -130,10 +130,6 @@ func (d *device) getLUID() winipcfg.LUID { return winipcfg.LUID(d.dev.(*tun.NativeTun).LUID()) } -func (d *device) index() uint32 { - return d.interfaceIndex -} - func (d *device) addSubnet(_ context.Context, subnet netip.Prefix) error { return d.getLUID().AddIPAddress(subnet) } From e02463afe70f270a970cae6a4a9e35ef1ac97772 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sun, 2 Feb 2025 08:22:21 +0100 Subject: [PATCH 03/61] release workflow fix Signed-off-by: Thomas Hallgren --- .github/workflows/release.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a59e67bf30..c38fb3d779 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -112,7 +112,7 @@ jobs: echo "prerelease=false" >> $GITHUB_OUTPUT fi - name: Create draft release - if: steps.semver_check.outputs.draft == true + if: ${{ steps.semver_check.outputs.draft == 'true' }} uses: ncipollo/release-action@v1 with: artifacts: "binaries-*/*" @@ -123,7 +123,7 @@ jobs: ## Draft Release For more information, visit our [installation docs](https://www.telepresence.io/docs/latest/quick-start/). - name: Create release - if: steps.semver_check.outputs.draft != true + if: ${{ steps.semver_check.outputs.draft == 'false' }} uses: ncipollo/release-action@v1 with: artifacts: "binaries-*/*" @@ -148,9 +148,9 @@ jobs: ![Assets](https://static.scarf.sh/a.png?x-pxid=d842651a-2e4d-465a-98e1-4808722c01ab) - uses: actions/checkout@v4 - if: steps.semver_check.outputs.make_latest == true + if: ${{ steps.semver_check.outputs.make_latest == 'true' }} - name: Update Homebrew - if: steps.semver_check.outputs.make_latest == true + if: ${{ steps.semver_check.outputs.make_latest == 'true' }} run: | v=${{ github.ref_name }} packaging/homebrew-package.sh "${v#v}" tel2oss "${{ vars.GH_BOT_USER }}" "${{ vars.GH_BOT_EMAIL }}" "${{ secrets.HOMEBREW_TAP_TOKEN }}" @@ -159,7 +159,7 @@ jobs: needs: - push-images - publish-release - if: needs.publish-release.semver_check.outputs.draft != true + if: ${{ needs.publish-release.semver_check.outputs.draft == 'false' }} strategy: fail-fast: false matrix: From 23cf7bc6a13ae025db0b88bf46d33750ddb63d62 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sun, 2 Feb 2025 15:41:28 +0100 Subject: [PATCH 04/61] Using the --proxy-via flag would sometimes cause connection timeouts. Typically, a `telepresence connect --proxy-via =` would fail with a "deadline exceeded" message when several workloads were present in the namespace, the one targeted by the proxy-via didn't yet have an agent installed, and other workloads had an agent. This was due to a race condition in the logic for the agent-based port-forwards in the root daemon. The conditions causing this race are now eliminated. Signed-off-by: Thomas Hallgren --- CHANGELOG.yml | 7 +++ docs/release-notes.md | 6 ++ docs/release-notes.mdx | 4 ++ pkg/client/agentpf/clients.go | 103 +++++++++++++++++++++------------- 4 files changed, 80 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.yml b/CHANGELOG.yml index 0a4006be64..529d5a07d8 100644 --- a/CHANGELOG.yml +++ b/CHANGELOG.yml @@ -36,6 +36,13 @@ items: - version: 2.21.3 date: (TBD) notes: + - type: bugfix + title: Using the --proxy-via flag would sometimes cause connection timeouts. + body: >- + Typically, a `telepresence connect --proxy-via =` would fail with a "deadline exceeded" + message when several workloads were present in the namespace, the one targeted by the proxy-via didn't yet + have an agent installed, and other workloads had an agent. This was due to a race condition in the logic + for the agent-based port-forwards in the root daemon. The conditions causing this race are now eliminated. - type: bugfix title: Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS. body: >- diff --git a/docs/release-notes.md b/docs/release-notes.md index 0d1960bd69..36a7a46cc4 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -2,6 +2,12 @@ [comment]: # (Code generated by relnotesgen. DO NOT EDIT.) # Telepresence Release Notes ## Version 2.21.3 +##
bugfix
Using the --proxy-via flag would sometimes cause connection timeouts.
+
+ +Typically, a `telepresence connect --proxy-via =` would fail with a "deadline exceeded" message when several workloads were present in the namespace, the one targeted by the proxy-via didn't yet have an agent installed, and other workloads had an agent. This was due to a race condition in the logic for the agent-based port-forwards in the root daemon. The conditions causing this race are now eliminated. +
+ ##
bugfix
Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS.
diff --git a/docs/release-notes.mdx b/docs/release-notes.mdx index 2690b298c0..66ee7889a1 100644 --- a/docs/release-notes.mdx +++ b/docs/release-notes.mdx @@ -8,6 +8,10 @@ import { Note, Title, Body } from '@site/src/components/ReleaseNotes' # Telepresence Release Notes ## Version 2.21.3 + + Using the --proxy-via flag would sometimes cause connection timeouts. + Typically, a `telepresence connect --proxy-via =` would fail with a "deadline exceeded" message when several workloads were present in the namespace, the one targeted by the proxy-via didn't yet have an agent installed, and other workloads had an agent. This was due to a race condition in the logic for the agent-based port-forwards in the root daemon. The conditions causing this race are now eliminated. + Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS. A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the TUN-device on macOS based clients. diff --git a/pkg/client/agentpf/clients.go b/pkg/client/agentpf/clients.go index 256da6b9fd..19cf59e9b4 100644 --- a/pkg/client/agentpf/clients.go +++ b/pkg/client/agentpf/clients.go @@ -36,9 +36,11 @@ type client struct { session *manager.SessionInfo info *manager.AgentPodInfo ready chan error + remove func() cancelClient context.CancelFunc cancelDialWatch context.CancelFunc tunnelCount int32 + infant bool } func (ac *client) String() string { @@ -49,17 +51,35 @@ func (ac *client) String() string { return fmt.Sprintf("%s.%s:%d", ai.PodName, ai.Namespace, ai.ApiPort) } -func (ac *client) Tunnel(ctx context.Context, opts ...grpc.CallOption) (tunnel.Client, error) { +func (ac *client) ensureConnect(ctx context.Context) error { + ac.Lock() + infant := ac.infant + if infant { + ac.infant = false + } + ac.Unlock() + if infant { + go ac.connect(ctx, func() { + ac.remove() + }) + } select { case err, ok := <-ac.ready: if ok { // Put error back on channel in case this Tunnel is used again before it's deleted. ac.ready <- err - return nil, err + return err } // ready channel is closed. We are ready to go. case <-ctx.Done(): - return nil, ctx.Err() + return ctx.Err() + } + return nil +} + +func (ac *client) Tunnel(ctx context.Context, opts ...grpc.CallOption) (tunnel.Client, error) { + if err := ac.ensureConnect(ctx); err != nil { + return nil, err } tc, err := ac.cli.Tunnel(ctx, opts...) if err != nil { @@ -101,7 +121,14 @@ func (ac *client) connect(ctx context.Context, deleteMe func()) { ac.cli = cli ac.cancelClient = func() { // Need to run this in a separate thread to avoid deadlock. - go conn.Close() + go func() { + ac.Lock() + conn.Close() + ac.cancelClient = nil + ac.cli = nil + ac.infant = true + ac.Unlock() + }() } intercepted := ac.info.Intercepted ac.Unlock() @@ -110,11 +137,11 @@ func (ac *client) connect(ctx context.Context, deleteMe func()) { } } -func (ac *client) busy() bool { +func (ac *client) dormant() bool { ac.Lock() - bzy := ac.cli == nil || ac.info.Intercepted || atomic.LoadInt32(&ac.tunnelCount) > 0 + dormant := !(ac.infant || ac.cli == nil || ac.info.Intercepted) && atomic.LoadInt32(&ac.tunnelCount) == 0 ac.Unlock() - return bzy + return dormant } func (ac *client) intercepted() bool { @@ -124,17 +151,21 @@ func (ac *client) intercepted() bool { return ret } -func (ac *client) cancel() { +func (ac *client) cancel() bool { ac.Lock() cc := ac.cancelClient cdw := ac.cancelDialWatch ac.Unlock() + didCancel := false if cc != nil { + didCancel = true cc() } if cdw != nil { + didCancel = true cdw() } + return didCancel } func (ac *client) setIntercepted(ctx context.Context, k string, status bool) { @@ -168,14 +199,8 @@ func (ac *client) setIntercepted(ctx context.Context, k string, status bool) { func (ac *client) startDialWatcher(ctx context.Context) error { // Not called from the startup go routine, so wait for that routine to finish - select { - case err, ok := <-ac.ready: - if ok { - return err - } - // ready channel is closed. We are ready to go. - case <-ctx.Done(): - return ctx.Err() + if err := ac.ensureConnect(ctx); err != nil { + return err } return ac.startDialWatcherReady(ctx) } @@ -314,11 +339,14 @@ func (s *clients) hasWaiterFor(info *manager.AgentPodInfo) bool { func (s *clients) WatchAgentPods(ctx context.Context, rmc manager.ManagerClient) error { dlog.Debug(ctx, "WatchAgentPods starting") defer func() { - dlog.Debugf(ctx, "WatchAgentPods ending with %d clients still active", s.clients.Size()) + activeCount := 0 s.clients.Range(func(_ string, ac *client) bool { - ac.cancel() + if ac.cancel() { + activeCount++ + } return true }) + dlog.Debugf(ctx, "WatchAgentPods ending with %d clients still active", activeCount) s.disabled.Store(true) }() backoff := 100 * time.Millisecond @@ -509,40 +537,35 @@ func (s *clients) updateClients(ctx context.Context, ais []*manager.AgentPodInfo ac := &client{ ready: make(chan error, 1), session: s.session, - info: ai, + remove: func() { + deleteClient(k) + }, + info: ai, + infant: true, } - go ac.connect(ctx, func() { - deleteClient(k) - }) + dlog.Debugf(ctx, "Adding agent pod %s (%s)", k, net.IP(ai.PodIp)) return ac, false }) } - // Add clients for newly arrived intercepts + // Add clients for newly arrived agents. for k, ai := range aim { - if ai.Intercepted || s.isProxyVIA(ai) || s.hasWaiterFor(ai) { - addClient(k, ai) - } + addClient(k, ai) } - // Terminate all non-intercepting idle agents except the last one. + // Terminate all dormant agents except the last one. + dormantCount := 0 s.clients.Range(func(k string, ac *client) bool { - if s.clients.Size() <= 1 { - return false - } - if !ac.busy() && !s.isProxyVIA(ac.info) && !s.hasWaiterFor(ac.info) { - deleteClient(k) + if ac.dormant() && !s.isProxyVIA(ac.info) && !s.hasWaiterFor(ac.info) { + dormantCount++ + if dormantCount > 1 { + ac.cancel() + } } return true }) - - // Ensure that we have at least one client (if at least one agent exists) - if s.clients.Size() == 0 { - for _, ai := range aim { - k := ai.PodName + "." + ai.Namespace - addClient(k, ai) - break - } + if dormantCount > 1 { + dlog.Debugf(ctx, "Cancelled %d dormant clients", dormantCount-1) } return nil } From 5a0825128741576d1f31344650b501f415b04f5c Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Fri, 10 Jan 2025 13:21:51 +0100 Subject: [PATCH 05/61] Ensure that the correct pod-log is captured after a helm rollback. Signed-off-by: Thomas Hallgren --- integration_test/itest/namespace.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration_test/itest/namespace.go b/integration_test/itest/namespace.go index 945e7e43fb..73a47a20f4 100644 --- a/integration_test/itest/namespace.go +++ b/integration_test/itest/namespace.go @@ -160,6 +160,9 @@ func (s *nsPair) RollbackTM(ctx context.Context) { t := getT(ctx) require.NoError(t, err) require.NoError(t, RolloutStatusWait(ctx, s.Namespace, "deploy/traffic-manager")) + assert.Eventually(t, func() bool { + return len(RunningPodNames(ctx, "traffic-manager", s.Namespace)) == 1 + }, 30*time.Second, 5*time.Second) s.CapturePodLogs(ctx, "traffic-manager", "", s.Namespace) } From b14bccc4dea421f79bce5fe9382864020cb818c7 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Mon, 3 Feb 2025 00:06:36 +0100 Subject: [PATCH 06/61] Ensure that make generate regenerates the docs files. Signed-off-by: Thomas Hallgren --- build-aux/main.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-aux/main.mk b/build-aux/main.mk index 699d94d010..aa93e02b0d 100644 --- a/build-aux/main.mk +++ b/build-aux/main.mk @@ -86,7 +86,7 @@ protoc: protoc-clean $(tools/protoc) $(tools/protoc-gen-go) $(tools/protoc-gen-g .PHONY: generate generate: ## (Generate) Update generated files that get checked in to Git generate: generate-clean -generate: protoc $(tools/go-mkopensource) $(BUILDDIR)/$(shell go env GOVERSION).src.tar.gz docs-files +generate: protoc $(tools/go-mkopensource) $(BUILDDIR)/$(shell go env GOVERSION).src.tar.gz cd ./rpc && export GOFLAGS=-mod=mod && go mod tidy && go mod vendor && rm -rf vendor cd ./pkg/vif/testdata/router && export GOFLAGS=-mod=mod && go mod tidy && go mod vendor && rm -rf vendor cd ./tools/src/test-report && export GOFLAGS=-mod=mod && go mod tidy && go mod vendor && rm -rf vendor @@ -108,12 +108,15 @@ generate: protoc $(tools/go-mkopensource) $(BUILDDIR)/$(shell go env GOVERSION). rm -rf vendor +generate: docs-files + .PHONY: generate-clean generate-clean: ## (Generate) Delete generated files rm -rf ./rpc/vendor rm -rf ./vendor rm -f DEPENDENCIES.md rm -f DEPENDENCY_LICENSES.md + rm -f docs/release-notes.md* CHANGELOG.yml: FORCE @# Check if the version is in the x.x.x format (GA release) From 1f733e7933c8fcd53700eb372883c6fb34335ed9 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Mon, 3 Feb 2025 00:15:02 +0100 Subject: [PATCH 07/61] Prepare v2.21.3-test.4 Signed-off-by: Thomas Hallgren --- go.mod | 2 +- pkg/vif/testdata/router/go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index ca138870d2..aa7d70e304 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.10.0 github.com/telepresenceio/go-fuseftp/rpc v0.5.0 - github.com/telepresenceio/telepresence/rpc/v2 v2.21.2 + github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-test.4 github.com/vishvananda/netlink v1.3.0 golang.org/x/exp v0.0.0-20241210194714-1829a127f884 golang.org/x/net v0.32.0 diff --git a/pkg/vif/testdata/router/go.mod b/pkg/vif/testdata/router/go.mod index d691070a0f..4ab32ff087 100644 --- a/pkg/vif/testdata/router/go.mod +++ b/pkg/vif/testdata/router/go.mod @@ -50,7 +50,7 @@ require ( github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/telepresenceio/telepresence/rpc/v2 v2.21.2 // indirect + github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-test.4 // indirect github.com/vishvananda/netlink v1.3.0 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect From 0566da495a232cb23a71db327dc4aba497eaf0b2 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Mon, 3 Feb 2025 07:34:57 +0100 Subject: [PATCH 08/61] Ensure that annotation enabled traffic-agents are uninstalled. Signed-off-by: Thomas Hallgren --- CHANGELOG.yml | 5 ++ cmd/traffic/cmd/manager/mutator/watcher.go | 63 ++++++++++++---------- docs/release-notes.md | 6 +++ docs/release-notes.mdx | 4 ++ 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.yml b/CHANGELOG.yml index 529d5a07d8..068b43fada 100644 --- a/CHANGELOG.yml +++ b/CHANGELOG.yml @@ -48,6 +48,11 @@ items: body: >- A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the TUN-device on macOS based clients. + - type: bugfix + title: Ensure that annotation enabled traffic-agents are uninstall when uninstalling the traffic-manager. + body: >- + A traffic-agent injected because the workload had the inject annotation enabled would sometimes not get + uninstalled when the traffic-manager was uninstalled. - version: 2.21.2 date: 2025-01-26 notes: diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index de1db3f4c1..103a542d58 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -6,6 +6,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/google/go-cmp/cmp" @@ -101,7 +102,7 @@ func (c *configWatcher) isRolloutNeeded(ctx context.Context, wl k8sapi.Workload, if ia, ok := podMeta.GetAnnotations()[agentconfig.InjectAnnotation]; ok { // Annotation controls injection, so no explicit rollout is needed unless the deployment was added before the traffic-manager. // If the annotation changes, there will be an implicit rollout anyway. - if wl.GetCreationTimestamp().After(c.startedAt) { + if wl.GetCreationTimestamp().After(c.startedAt) && c.running.Load() { dlog.Debugf(ctx, "Rollout of %s.%s is not necessary. Pod template has inject annotation %s", wl.GetName(), wl.GetNamespace(), ia) return false @@ -424,6 +425,7 @@ type configWatcher struct { nsLocks *xsync.MapOf[string, *sync.RWMutex] blacklistedPods *xsync.MapOf[string, time.Time] startedAt time.Time + running atomic.Bool rolloutDisabled bool cms []cache.SharedIndexInformer @@ -538,6 +540,7 @@ func (c *configWatcher) SetSelf(self Map) { } func (c *configWatcher) StartWatchers(ctx context.Context) error { + defer c.running.Store(true) c.startedAt = time.Now() ctx, c.cancel = context.WithCancel(ctx) for _, si := range c.svs { @@ -857,36 +860,38 @@ func (c *configWatcher) Start(ctx context.Context) { } func (c *configWatcher) DeleteMapsAndRolloutAll(ctx context.Context) { - c.cancel() // No more updates from watcher - now := meta.NewDeleteOptions(0) - api := k8sapi.GetK8sInterface(ctx).CoreV1() - c.nsLocks.Range(func(ns string, lock *sync.RWMutex) bool { - lock.Lock() - defer lock.Unlock() - wlm, err := data(ctx, ns) - if err != nil { - dlog.Errorf(ctx, "unable to get configmap %s.%s: %v", agentconfig.ConfigMap, ns, err) - return true - } - for k, v := range wlm { - e := &entry{name: k, namespace: ns, value: v} - scx, wl, err := e.workload(ctx) + if c.running.CompareAndSwap(true, false) { + c.cancel() // No more updates from watcher + now := meta.NewDeleteOptions(0) + api := k8sapi.GetK8sInterface(ctx).CoreV1() + c.nsLocks.Range(func(ns string, lock *sync.RWMutex) bool { + lock.Lock() + defer lock.Unlock() + wlm, err := data(ctx, ns) if err != nil { - if !errors.IsNotFound(err) { - dlog.Errorf(ctx, "unable to get workload for %s.%s %s: %v", k, ns, v, err) + dlog.Errorf(ctx, "unable to get configmap %s.%s: %v", agentconfig.ConfigMap, ns, err) + return true + } + for k, v := range wlm { + e := &entry{name: k, namespace: ns, value: v} + scx, wl, err := e.workload(ctx) + if err != nil { + if !errors.IsNotFound(err) { + dlog.Errorf(ctx, "unable to get workload for %s.%s %s: %v", k, ns, v, err) + } + continue } - continue + ac := scx.AgentConfig() + if ac.Create || ac.Manual { + // Deleted before it was generated or manually added, just ignore + continue + } + c.triggerRollout(ctx, wl, nil) } - ac := scx.AgentConfig() - if ac.Create || ac.Manual { - // Deleted before it was generated or manually added, just ignore - continue + if err := api.ConfigMaps(ns).Delete(ctx, agentconfig.ConfigMap, *now); err != nil { + dlog.Errorf(ctx, "unable to delete ConfigMap %s-%s: %v", agentconfig.ConfigMap, ns, err) } - c.triggerRollout(ctx, wl, nil) - } - if err := api.ConfigMaps(ns).Delete(ctx, agentconfig.ConfigMap, *now); err != nil { - dlog.Errorf(ctx, "unable to delete ConfigMap %s-%s: %v", agentconfig.ConfigMap, ns, err) - } - return true - }) + return true + }) + } } diff --git a/docs/release-notes.md b/docs/release-notes.md index 36a7a46cc4..d8fdab81b7 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -14,6 +14,12 @@ Typically, a `telepresence connect --proxy-via =` would fail w A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the TUN-device on macOS based clients.
+##
bugfix
Ensure that annotation enabled traffic-agents are uninstall when uninstalling the traffic-manager.
+
+ +A traffic-agent injected because the workload had the inject annotation enabled would sometimes not get uninstalled when the traffic-manager was uninstalled. +
+ ## Version 2.21.2 (January 26) ##
bugfix
Fix panic when agentpf.client creates a Tunnel
diff --git a/docs/release-notes.mdx b/docs/release-notes.mdx index 66ee7889a1..13ad2bc0d6 100644 --- a/docs/release-notes.mdx +++ b/docs/release-notes.mdx @@ -16,6 +16,10 @@ import { Note, Title, Body } from '@site/src/components/ReleaseNotes' Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS. A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the TUN-device on macOS based clients. + + Ensure that annotation enabled traffic-agents are uninstall when uninstalling the traffic-manager. + A traffic-agent injected because the workload had the inject annotation enabled would sometimes not get uninstalled when the traffic-manager was uninstalled. + ## Version 2.21.2 (January 26) Fix panic when agentpf.client creates a Tunnel From 7a4a4d87019e2728870f06f40d3336e672f302ac Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 4 Feb 2025 16:45:11 +0100 Subject: [PATCH 09/61] Prefer route with gateway when adding static route. Signed-off-by: Thomas Hallgren --- pkg/vif/router.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/vif/router.go b/pkg/vif/router.go index 2f40bbb265..8dfbf95605 100644 --- a/pkg/vif/router.go +++ b/pkg/vif/router.go @@ -191,12 +191,22 @@ func (rt *Router) addStaticOverrides(ctx context.Context, neverProxy, neverProxy if err != nil { return err } - pr = &routing.Route{ - Interface: ifd, + for _, addr := range addrs { + pfx, err := netip.ParsePrefix(addr.String()) + if err != nil { + return err + } + pr, err = routing.GetRoute(ctx, pfx) + if err != nil { + return err + } + if pr.Gateway.IsValid() { + break + } } - if len(addrs) > 0 { - if pfx, err := netip.ParsePrefix(addrs[0].String()); err == nil { - pr.LocalIP = pfx.Addr() + if pr == nil { + pr = &routing.Route{ + Interface: ifd, } } } From 734316bc499689fbf8d5b057638db5ae7da28404 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 4 Feb 2025 18:14:51 +0100 Subject: [PATCH 10/61] Never chose route with different IP family. Signed-off-by: Thomas Hallgren --- pkg/vif/router.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/pkg/vif/router.go b/pkg/vif/router.go index 8dfbf95605..3c599ad6b6 100644 --- a/pkg/vif/router.go +++ b/pkg/vif/router.go @@ -178,29 +178,34 @@ func (rt *Router) addStaticOverrides(ctx context.Context, neverProxy, neverProxy } } - if len(staticNets) > 0 { - ifd, err := net.InterfaceByIndex(int(rt.device.Index())) - if err != nil { - return err - } + ifd, err := net.InterfaceByIndex(int(rt.device.Index())) + if err != nil { + return err + } + addrs, err := ifd.Addrs() + if err != nil { + return err + } + for _, sn := range staticNets { var pr *routing.Route - if dr.Interface.Index == ifd.Index { + ip4 := sn.Addr().Is4() + if dr.Interface.Index == ifd.Index && ip4 == dr.LocalIP.Is4() { pr = dr } else { - addrs, err := ifd.Addrs() - if err != nil { - return err - } for _, addr := range addrs { pfx, err := netip.ParsePrefix(addr.String()) if err != nil { return err } + if ip4 != pfx.Addr().Is4() { + continue + } pr, err = routing.GetRoute(ctx, pfx) if err != nil { return err } if pr.Gateway.IsValid() { + // Address families match and we have a gateway. It doesn't get any better. break } } @@ -209,9 +214,7 @@ func (rt *Router) addStaticOverrides(ctx context.Context, neverProxy, neverProxy Interface: ifd, } } - } - for _, sn := range staticNets { desired = append(desired, &routing.Route{ LocalIP: pr.LocalIP, Gateway: pr.Gateway, From 1f662f5bfd8afaeadc9a9445755b6b8fab07cba9 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 4 Feb 2025 22:53:20 +0100 Subject: [PATCH 11/61] Prepare v2.21.3-test.5 Signed-off-by: Thomas Hallgren --- go.mod | 2 +- pkg/vif/testdata/router/go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index aa7d70e304..c1c45d7b3c 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.10.0 github.com/telepresenceio/go-fuseftp/rpc v0.5.0 - github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-test.4 + github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-test.5 github.com/vishvananda/netlink v1.3.0 golang.org/x/exp v0.0.0-20241210194714-1829a127f884 golang.org/x/net v0.32.0 diff --git a/pkg/vif/testdata/router/go.mod b/pkg/vif/testdata/router/go.mod index 4ab32ff087..f0373dea62 100644 --- a/pkg/vif/testdata/router/go.mod +++ b/pkg/vif/testdata/router/go.mod @@ -50,7 +50,7 @@ require ( github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-test.4 // indirect + github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-test.5 // indirect github.com/vishvananda/netlink v1.3.0 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect From 939a51d166a2eb5db177c6d1bab53820427c01fa Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Wed, 5 Feb 2025 11:30:28 +0100 Subject: [PATCH 12/61] Prepare v2.21.3-rc.0 Signed-off-by: Thomas Hallgren --- go.mod | 2 +- pkg/vif/testdata/router/go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index c1c45d7b3c..c03fae4b0f 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.10.0 github.com/telepresenceio/go-fuseftp/rpc v0.5.0 - github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-test.5 + github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-rc.0 github.com/vishvananda/netlink v1.3.0 golang.org/x/exp v0.0.0-20241210194714-1829a127f884 golang.org/x/net v0.32.0 diff --git a/pkg/vif/testdata/router/go.mod b/pkg/vif/testdata/router/go.mod index f0373dea62..19457efeb8 100644 --- a/pkg/vif/testdata/router/go.mod +++ b/pkg/vif/testdata/router/go.mod @@ -50,7 +50,7 @@ require ( github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-test.5 // indirect + github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-rc.0 // indirect github.com/vishvananda/netlink v1.3.0 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect From 2e00a411d381801861e32b22cce5b5714ca92237 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Thu, 6 Feb 2025 07:23:09 +0100 Subject: [PATCH 13/61] Prepare v2.21.3 Signed-off-by: Thomas Hallgren --- CHANGELOG.yml | 2 +- docs/release-notes.md | 2 +- docs/release-notes.mdx | 2 +- go.mod | 2 +- pkg/vif/testdata/router/go.mod | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.yml b/CHANGELOG.yml index 068b43fada..a3e7b6aa69 100644 --- a/CHANGELOG.yml +++ b/CHANGELOG.yml @@ -34,7 +34,7 @@ docDescription: >- customizable development environments. items: - version: 2.21.3 - date: (TBD) + date: 2025-02-06 notes: - type: bugfix title: Using the --proxy-via flag would sometimes cause connection timeouts. diff --git a/docs/release-notes.md b/docs/release-notes.md index d8fdab81b7..dec30b5f72 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,7 +1,7 @@ [comment]: # (Code generated by relnotesgen. DO NOT EDIT.) # Telepresence Release Notes -## Version 2.21.3 +## Version 2.21.3 (February 6) ##
bugfix
Using the --proxy-via flag would sometimes cause connection timeouts.
diff --git a/docs/release-notes.mdx b/docs/release-notes.mdx index 13ad2bc0d6..0224c550e6 100644 --- a/docs/release-notes.mdx +++ b/docs/release-notes.mdx @@ -7,7 +7,7 @@ import { Note, Title, Body } from '@site/src/components/ReleaseNotes' [comment]: # (Code generated by relnotesgen. DO NOT EDIT.) # Telepresence Release Notes -## Version 2.21.3 +## Version 2.21.3 (February 6) Using the --proxy-via flag would sometimes cause connection timeouts. Typically, a `telepresence connect --proxy-via =` would fail with a "deadline exceeded" message when several workloads were present in the namespace, the one targeted by the proxy-via didn't yet have an agent installed, and other workloads had an agent. This was due to a race condition in the logic for the agent-based port-forwards in the root daemon. The conditions causing this race are now eliminated. diff --git a/go.mod b/go.mod index c03fae4b0f..ad720d3368 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.10.0 github.com/telepresenceio/go-fuseftp/rpc v0.5.0 - github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-rc.0 + github.com/telepresenceio/telepresence/rpc/v2 v2.21.3 github.com/vishvananda/netlink v1.3.0 golang.org/x/exp v0.0.0-20241210194714-1829a127f884 golang.org/x/net v0.32.0 diff --git a/pkg/vif/testdata/router/go.mod b/pkg/vif/testdata/router/go.mod index 19457efeb8..ee0d7960ae 100644 --- a/pkg/vif/testdata/router/go.mod +++ b/pkg/vif/testdata/router/go.mod @@ -50,7 +50,7 @@ require ( github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/telepresenceio/telepresence/rpc/v2 v2.21.3-rc.0 // indirect + github.com/telepresenceio/telepresence/rpc/v2 v2.21.3 // indirect github.com/vishvananda/netlink v1.3.0 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect From 3092af20ee850afb3513b5ce612cb4090649c5af Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Wed, 8 Jan 2025 18:32:22 +0100 Subject: [PATCH 14/61] Base connectedSuite on TrafficManager instead of NamespacePair. Signed-off-by: Thomas Hallgren --- integration_test/connected_test.go | 6 ++-- integration_test/itest/cluster.go | 2 +- integration_test/itest/connected.go | 6 ++-- integration_test/itest/runner.go | 10 +++---- integration_test/itest/template.go | 15 ++++++++++ integration_test/manager_grpc_test.go | 6 ++-- integration_test/mounts_test.go | 8 ++--- integration_test/testdata/k8s/generic.goyaml | 31 +++++++++++++++++--- integration_test/webhook_test.go | 6 ++-- integration_test/workspace_watch_test.go | 2 ++ 10 files changed, 66 insertions(+), 26 deletions(-) diff --git a/integration_test/connected_test.go b/integration_test/connected_test.go index 1445e0cc64..09b014afca 100644 --- a/integration_test/connected_test.go +++ b/integration_test/connected_test.go @@ -9,7 +9,7 @@ import ( type connectedSuite struct { itest.Suite - itest.NamespacePair + itest.TrafficManager } func (s *connectedSuite) SuiteName() string { @@ -17,8 +17,8 @@ func (s *connectedSuite) SuiteName() string { } func init() { - itest.AddConnectedSuite("", func(h itest.NamespacePair) itest.TestingSuite { - return &connectedSuite{Suite: itest.Suite{Harness: h}, NamespacePair: h} + itest.AddConnectedSuite("", func(h itest.TrafficManager) itest.TestingSuite { + return &connectedSuite{Suite: itest.Suite{Harness: h}, TrafficManager: h} }) } diff --git a/integration_test/itest/cluster.go b/integration_test/itest/cluster.go index 8ed742f2ea..c40a72504f 100644 --- a/integration_test/itest/cluster.go +++ b/integration_test/itest/cluster.go @@ -909,7 +909,7 @@ func TelepresenceOk(ctx context.Context, args ...string) string { stdout, stderr, err := Telepresence(ctx, args...) require.NoError(t, err, "telepresence was unable to run, stdout %s", stdout) if err == nil { - if strings.HasPrefix(stderr, "Warning:") && !strings.ContainsRune(stderr, '\n') { + if (strings.HasPrefix(stderr, "Warning:") || strings.Contains(stderr, "has been deprecated")) && !strings.ContainsRune(stderr, '\n') { // Accept warnings, but log them. dlog.Warn(ctx, stderr) } else { diff --git a/integration_test/itest/connected.go b/integration_test/itest/connected.go index 8551c94f2f..7053368fc9 100644 --- a/integration_test/itest/connected.go +++ b/integration_test/itest/connected.go @@ -9,14 +9,14 @@ import ( ) type connected struct { - NamespacePair + TrafficManager } -func WithConnected(np NamespacePair, f func(ctx context.Context, ch NamespacePair)) { +func WithConnected(np TrafficManager, f func(ctx context.Context, ch TrafficManager)) { np.HarnessT().Run("Test_Connected", func(t *testing.T) { ctx := WithT(np.HarnessContext(), t) require.NoError(t, np.GeneralError()) - ch := &connected{NamespacePair: np} + ch := &connected{TrafficManager: np} ch.PushHarness(ctx, ch.setup, ch.tearDown) defer ch.PopHarness() f(ctx, ch) diff --git a/integration_test/itest/runner.go b/integration_test/itest/runner.go index 7ac8e4318a..7cc5ca3cfa 100644 --- a/integration_test/itest/runner.go +++ b/integration_test/itest/runner.go @@ -13,7 +13,7 @@ type Runner interface { AddClusterSuite(func(context.Context) TestingSuite) AddNamespacePairSuite(suffix string, f func(NamespacePair) TestingSuite) AddTrafficManagerSuite(suffix string, f func(TrafficManager) TestingSuite) - AddConnectedSuite(suffix string, f func(NamespacePair) TestingSuite) + AddConnectedSuite(suffix string, f func(TrafficManager) TestingSuite) AddMultipleServicesSuite(suffix, name string, f func(MultipleServices) TestingSuite) AddSingleServiceSuite(suffix, name string, f func(SingleService) TestingSuite) RunTests(context.Context) @@ -27,7 +27,7 @@ type namedRunner struct { type suffixedRunner struct { withNamespace []func(NamespacePair) TestingSuite withTrafficManager []func(TrafficManager) TestingSuite - withConnected []func(NamespacePair) TestingSuite + withConnected []func(TrafficManager) TestingSuite withName map[string]*namedRunner } @@ -94,13 +94,13 @@ func (r *suffixedRunner) forName(name string) *namedRunner { // AddConnectedSuite adds a constructor for a test suite to the default runner that requires a cluster where a namespace // pair has been initialized, and telepresence is connected. -func AddConnectedSuite(suffix string, f func(NamespacePair) TestingSuite) { +func AddConnectedSuite(suffix string, f func(TrafficManager) TestingSuite) { defaultRunner.AddConnectedSuite(suffix, f) } // AddConnectedSuite adds a constructor for a test suite to the default runner that requires a cluster where a namespace // pair has been initialized, and telepresence is connected. -func (r *runner) AddConnectedSuite(suffix string, f func(NamespacePair) TestingSuite) { +func (r *runner) AddConnectedSuite(suffix string, f func(TrafficManager) TestingSuite) { sr := r.forSuffix(suffix) sr.withConnected = append(sr.withConnected, f) } @@ -164,7 +164,7 @@ func (r *runner) RunTests(c context.Context) { //nolint:gocognit cnp.RunSuite(f(cnp)) } if len(sr.withConnected)+len(sr.withName) > 0 { - WithConnected(np, func(c context.Context, cnp NamespacePair) { + WithConnected(cnp, func(c context.Context, cnp TrafficManager) { for _, f := range sr.withConnected { cnp.RunSuite(f(cnp)) } diff --git a/integration_test/itest/template.go b/integration_test/itest/template.go index 6404242010..047e13123d 100644 --- a/integration_test/itest/template.go +++ b/integration_test/itest/template.go @@ -13,12 +13,27 @@ import ( "sigs.k8s.io/yaml" ) +type ContainerPort struct { + Number int + Name string + Protocol core.Protocol +} + +type ServicePort struct { + Number int + Name string + Protocol core.Protocol + TargetPort string +} + type Generic struct { Name string Annotations map[string]string Environment []core.EnvVar TargetPort string + ServicePorts []ServicePort ContainerPort int + ContainerPorts []ContainerPort Image string Registry string ServiceAccount string diff --git a/integration_test/manager_grpc_test.go b/integration_test/manager_grpc_test.go index ca519a5a17..e0b9e654d8 100644 --- a/integration_test/manager_grpc_test.go +++ b/integration_test/manager_grpc_test.go @@ -16,7 +16,7 @@ import ( type managerGRPCSuite struct { itest.Suite - itest.NamespacePair + itest.TrafficManager conn *grpc.ClientConn client manager.ManagerClient si *manager.SessionInfo @@ -27,8 +27,8 @@ func (m *managerGRPCSuite) SuiteName() string { } func init() { - itest.AddConnectedSuite("", func(h itest.NamespacePair) itest.TestingSuite { - return &managerGRPCSuite{Suite: itest.Suite{Harness: h}, NamespacePair: h} + itest.AddConnectedSuite("", func(h itest.TrafficManager) itest.TestingSuite { + return &managerGRPCSuite{Suite: itest.Suite{Harness: h}, TrafficManager: h} }) } diff --git a/integration_test/mounts_test.go b/integration_test/mounts_test.go index 38e5210df8..5fba1612d5 100644 --- a/integration_test/mounts_test.go +++ b/integration_test/mounts_test.go @@ -18,7 +18,7 @@ import ( type mountsSuite struct { itest.Suite - itest.NamespacePair + itest.TrafficManager eksClusterName string } @@ -27,10 +27,10 @@ func (s *mountsSuite) SuiteName() string { } func init() { - itest.AddConnectedSuite("", func(h itest.NamespacePair) itest.TestingSuite { + itest.AddConnectedSuite("", func(h itest.TrafficManager) itest.TestingSuite { return &mountsSuite{ - Suite: itest.Suite{Harness: h}, - NamespacePair: h, + Suite: itest.Suite{Harness: h}, + TrafficManager: h, } }) } diff --git a/integration_test/testdata/k8s/generic.goyaml b/integration_test/testdata/k8s/generic.goyaml index 5a7f914d53..6a33f9c325 100644 --- a/integration_test/testdata/k8s/generic.goyaml +++ b/integration_test/testdata/k8s/generic.goyaml @@ -9,9 +9,22 @@ spec: selector: app: {{ .Name }} ports: - - name: http - port: 80 +{{- range .ServicePorts }} + - port: {{ .Number }} +{{- with .Name }} + name: {{ . }} +{{- end}} +{{- with .TargetPort }} + targetPort: {{ . }} +{{- end}} +{{- with .Protocol }} + protocol: {{ . }} +{{- end}} +{{- else }} + - port: 80 + name: http targetPort: {{ .TargetPort | default "http" }} +{{- end}} --- apiVersion: apps/v1 kind: Deployment @@ -37,8 +50,18 @@ spec: - name: backend image: "{{ .Registry }}/{{ .Image }}" ports: - - name: http - containerPort: {{ .ContainerPort | default 8080 }} +{{- range .ContainerPorts }} + - containerPort: {{ .Number }} + {{- with .Name }} + name: {{ . }} + {{- end}} + {{- with .Protocol }} + protocol: {{ . }} + {{- end}} +{{- else }} + - containerPort: {{ .ContainerPort | default 8080 }} + name: http +{{- end }} {{- with .Environment }} env: {{- toYaml . | nindent 12 }} diff --git a/integration_test/webhook_test.go b/integration_test/webhook_test.go index 5b1c9675d1..6f21ff78a1 100644 --- a/integration_test/webhook_test.go +++ b/integration_test/webhook_test.go @@ -10,7 +10,7 @@ import ( type webhookSuite struct { itest.Suite - itest.NamespacePair + itest.TrafficManager } func (s *webhookSuite) SuiteName() string { @@ -18,8 +18,8 @@ func (s *webhookSuite) SuiteName() string { } func init() { - itest.AddConnectedSuite("", func(h itest.NamespacePair) itest.TestingSuite { - return &webhookSuite{Suite: itest.Suite{Harness: h}, NamespacePair: h} + itest.AddConnectedSuite("", func(h itest.TrafficManager) itest.TestingSuite { + return &webhookSuite{Suite: itest.Suite{Harness: h}, TrafficManager: h} }) } diff --git a/integration_test/workspace_watch_test.go b/integration_test/workspace_watch_test.go index c674b971db..7c0b36b6c3 100644 --- a/integration_test/workspace_watch_test.go +++ b/integration_test/workspace_watch_test.go @@ -43,6 +43,8 @@ func (s *notConnectedSuite) Test_WorkspaceListener() { spec.ServicePort = pi.ServicePort spec.ServicePortName = pi.ServicePortName spec.ServiceUid = pi.ServiceUid + spec.ContainerPort = pi.ContainerPort + spec.ContainerName = pi.ContainerName if pi.ServiceUid != "" { if pi.ServicePortName != "" { spec.PortIdentifier = pi.ServicePortName From a1554c00725c9be75a1cd209a54d1e782f4f83f3 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Wed, 8 Jan 2025 23:42:29 +0100 Subject: [PATCH 15/61] Using the --namespace option with telepresence causes a deadlock. Using `telepresence list --namespace with a namespace different from the one that telepresence was connected to, would cause a deadlock, and then produce an empty list. Signed-off-by: Thomas Hallgren --- CHANGELOG.yml | 7 +++- cmd/traffic/cmd/manager/manager.go | 22 ++++++------- cmd/traffic/cmd/manager/service.go | 2 +- cmd/traffic/cmd/manager/state/state.go | 9 ++---- .../manager/state/workload_info_watcher.go | 2 +- docs/release-notes.md | 32 ++----------------- docs/release-notes.mdx | 24 ++------------ pkg/informer/context.go | 13 +++++++- 8 files changed, 39 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.yml b/CHANGELOG.yml index 9fce39e855..0aebb27d58 100644 --- a/CHANGELOG.yml +++ b/CHANGELOG.yml @@ -25,7 +25,7 @@ # For older changes, see CHANGELOG.OLD.md items: - version: 2.22.0 - date: 2025-02-06 + date: (TBD) notes: - type: feature title: One single invocation of the Telepresence intercept command can now intercept multiple ports. @@ -75,6 +75,11 @@ items: body: >- macOS based systems will often PTR queries using nameslike `b._dns-sd._udp`, lb._dns-sd._udp, or `db-dns-sd._udp`. Those queries are no longer dispatched to the cluster. + - type: bugfix + title: Using the --namespace option with telepresence causes a deadlock. + body: >- + Using `telepresence list --namespace with a namespace different from the one that telepresence was + connected to, would cause a deadlock, and then produce an empty list. - version: 2.21.3 date: 2025-02-06 notes: diff --git a/cmd/traffic/cmd/manager/manager.go b/cmd/traffic/cmd/manager/manager.go index 584e5ae781..7d2ba50b0d 100644 --- a/cmd/traffic/cmd/manager/manager.go +++ b/cmd/traffic/cmd/manager/manager.go @@ -106,17 +106,17 @@ func MainWithEnv(ctx context.Context) (err error) { // l := klog.Level(6) // _ = l.Set("6") mgrFactory := false - mns := namespaces.Get(ctx) - if len(mns) == 0 { - ctx = informer.WithFactory(ctx, "") - } else { - for _, ns := range mns { - ctx = informer.WithFactory(ctx, ns) - } - if !slices.Contains(mns, env.ManagerNamespace) { - mgrFactory = true - ctx = informer.WithFactory(ctx, env.ManagerNamespace) - } + mns := namespaces.GetOrGlobal(ctx) + global := len(mns) == 1 && mns[0] == "" + if global { + dlog.Debug(ctx, "Using cluster wide informers") + } + for _, ns := range mns { + ctx = informer.WithFactory(ctx, ns) + } + if !(global || slices.Contains(mns, env.ManagerNamespace)) { + mgrFactory = true + ctx = informer.WithFactory(ctx, env.ManagerNamespace) } var injectorCertGetter mutator.InjectorCertGetter diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index 2d6d75bf41..ce0cd32c3f 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -937,7 +937,7 @@ func (s *service) WatchWorkloads(request *rpc.WorkloadEventsRequest, stream rpc. } dlog.Debugf(ctx, "WatchWorkloads ended") }() - dlog.Debugf(ctx, "WatchWorkloads called") + dlog.Debugf(ctx, "WatchWorkloads called, namespace %q", request.Namespace) if request.SessionInfo == nil { return status.Error(codes.InvalidArgument, "SessionInfo is required") diff --git a/cmd/traffic/cmd/manager/state/state.go b/cmd/traffic/cmd/manager/state/state.go index fe54e9c9c1..5f4d7e074e 100644 --- a/cmd/traffic/cmd/manager/state/state.go +++ b/cmd/traffic/cmd/manager/state/state.go @@ -90,7 +90,7 @@ type State interface { WatchAgents(context.Context, func(sessionID string, agent *rpc.AgentInfo) bool) <-chan watchable.Snapshot[*rpc.AgentInfo] WatchDial(sessionID string) <-chan *rpc.DialRequest WatchIntercepts(context.Context, func(sessionID string, intercept *rpc.InterceptInfo) bool) <-chan watchable.Snapshot[*rpc.InterceptInfo] - WatchWorkloads(ctx context.Context, sessionID string) (ch <-chan []workload.Event, err error) + WatchWorkloads(ctx context.Context, namespace string) (ch <-chan []workload.Event, err error) WatchLookupDNS(string) <-chan *rpc.DNSRequest ValidateCreateAgent(context.Context, k8sapi.Workload, agentconfig.SidecarExt) error NewWorkloadInfoWatcher(clientSession, namespace string) WorkloadInfoWatcher @@ -522,12 +522,7 @@ func (s *state) WatchAgents( } } -func (s *state) WatchWorkloads(ctx context.Context, sessionID string) (ch <-chan []workload.Event, err error) { - client := s.GetClient(sessionID) - if client == nil { - return nil, status.Errorf(codes.NotFound, "session %q not found", sessionID) - } - ns := client.Namespace +func (s *state) WatchWorkloads(ctx context.Context, ns string) (ch <-chan []workload.Event, err error) { ww, _ := s.workloadWatchers.Compute(ns, func(ww workload.Watcher, loaded bool) (workload.Watcher, bool) { if loaded { return ww, false diff --git a/cmd/traffic/cmd/manager/state/workload_info_watcher.go b/cmd/traffic/cmd/manager/state/workload_info_watcher.go index 6198816a13..65d059128b 100644 --- a/cmd/traffic/cmd/manager/state/workload_info_watcher.go +++ b/cmd/traffic/cmd/manager/state/workload_info_watcher.go @@ -62,7 +62,7 @@ func (wf *workloadInfoWatcher) Watch(ctx context.Context, stream rpc.Manager_Wat return err } - workloadsCh, err := wf.WatchWorkloads(ctx, wf.clientSession) + workloadsCh, err := wf.WatchWorkloads(ctx, wf.namespace) if err != nil { return err } diff --git a/docs/release-notes.md b/docs/release-notes.md index 2c4b9d1776..0c08047d8f 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,7 +1,7 @@ [comment]: # (Code generated by relnotesgen. DO NOT EDIT.) # Telepresence Release Notes -## Version 2.22.0 (February 6) +## Version 2.22.0 ##
feature
One single invocation of the Telepresence intercept command can now intercept multiple ports.
@@ -56,36 +56,10 @@ The namespace conflict detection mechanism would only discover conflicts between macOS based systems will often PTR queries using nameslike `b._dns-sd._udp`, lb._dns-sd._udp, or `db-dns-sd._udp`. Those queries are no longer dispatched to the cluster.
-## Version 2.21.3 (February 6) -##
bugfix
Using the --proxy-via flag would sometimes cause connection timeouts.
+##
bugfix
Using the --namespace option with telepresence causes a deadlock.
-Typically, a `telepresence connect --proxy-via =` would fail with a "deadline exceeded" message when several workloads were present in the namespace, the one targeted by the proxy-via didn't yet have an agent installed, and other workloads had an agent. This was due to a race condition in the logic for the agent-based port-forwards in the root daemon. The conditions causing this race are now eliminated. -
- -##
bugfix
Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS.
-
- -A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the TUN-device on macOS based clients. -
- -##
bugfix
Ensure that annotation enabled traffic-agents are uninstall when uninstalling the traffic-manager.
-
- -A traffic-agent injected because the workload had the inject annotation enabled would sometimes not get uninstalled when the traffic-manager was uninstalled. -
- -## Version 2.21.2 (January 26) -##
bugfix
Fix panic when agentpf.client creates a Tunnel
-
- -A race could occur where several requests where made to `agentpf.client.Tunnel` on a client that had errored when creating its port-forward to the agent. The implementation could handle one such requests but not several, resulting in a panic in situations where multiple simultaneous requests were made to the same client during a very short time period, -
- -##
bugfix
Fix goroutine leak in dialer.
-
- -The context passed to the `Tunnel` call that creates a stream for a dialer, was not cancelled when the dialer was finished, so the stream was never properly closed, leading to one dormant goroutine for each stream. +Using `telepresence list --namespace with a namespace different from the one that telepresence was connected to, would cause a deadlock, and then produce an empty list.
## Version 2.21.1 (December 17) diff --git a/docs/release-notes.mdx b/docs/release-notes.mdx index e8132fa986..7f196dbffc 100644 --- a/docs/release-notes.mdx +++ b/docs/release-notes.mdx @@ -7,7 +7,7 @@ import { Note, Title, Body } from '@site/src/components/ReleaseNotes' [comment]: # (Code generated by relnotesgen. DO NOT EDIT.) # Telepresence Release Notes -## Version 2.22.0 (February 6) +## Version 2.22.0 One single invocation of the Telepresence intercept command can now intercept multiple ports. It is now possible to intercept multiple ports with one single invocation of `telepresence intercept` by just repeating the `--port` flag. @@ -50,27 +50,9 @@ traffic to the original application container. This simplification offers severa Don't dispatch DNS discovery queries to the cluster. macOS based systems will often PTR queries using nameslike `b._dns-sd._udp`, lb._dns-sd._udp, or `db-dns-sd._udp`. Those queries are no longer dispatched to the cluster. -## Version 2.21.3 (February 6) - Using the --proxy-via flag would sometimes cause connection timeouts. - Typically, a `telepresence connect --proxy-via =` would fail with a "deadline exceeded" message when several workloads were present in the namespace, the one targeted by the proxy-via didn't yet have an agent installed, and other workloads had an agent. This was due to a race condition in the logic for the agent-based port-forwards in the root daemon. The conditions causing this race are now eliminated. - - - Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS. - A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the TUN-device on macOS based clients. - - - Ensure that annotation enabled traffic-agents are uninstall when uninstalling the traffic-manager. - A traffic-agent injected because the workload had the inject annotation enabled would sometimes not get uninstalled when the traffic-manager was uninstalled. - -## Version 2.21.2 (January 26) - - Fix panic when agentpf.client creates a Tunnel - A race could occur where several requests where made to `agentpf.client.Tunnel` on a client that had errored when creating its port-forward to the agent. The implementation could handle one such requests but not several, resulting in a panic in situations where multiple simultaneous requests were made to the same client during a very short time period, - - - Fix goroutine leak in dialer. - The context passed to the `Tunnel` call that creates a stream for a dialer, was not cancelled when the dialer was finished, so the stream was never properly closed, leading to one dormant goroutine for each stream. + Using the --namespace option with telepresence causes a deadlock. + Using `telepresence list --namespace with a namespace different from the one that telepresence was connected to, would cause a deadlock, and then produce an empty list. ## Version 2.21.1 (December 17) diff --git a/pkg/informer/context.go b/pkg/informer/context.go index cfd06d5653..e01a5c532e 100644 --- a/pkg/informer/context.go +++ b/pkg/informer/context.go @@ -21,9 +21,14 @@ func getOpts(ns string) (k8sOpts []informers.SharedInformerOption, argoOpts []ar return k8sOpts, argoOpts } -func WithFactory(ctx context.Context, _ string) context.Context { +func WithFactory(ctx context.Context, ns string) context.Context { if _, ok := ctx.Value(factoryKey{}).(*xsync.MapOf[string, GlobalFactory]); !ok { ctx = context.WithValue(ctx, factoryKey{}, xsync.NewMapOf[string, GlobalFactory]()) + if ns == "" { + // The cluster wide informer must be created when it is requested as the initial informer because it will act as a + // proxy for all other requested informers. + GetFactory(ctx, ns) + } } return ctx } @@ -34,6 +39,12 @@ func GetFactory(ctx context.Context, ns string) GlobalFactory { return nil } gf, _ := fm.LoadOrCompute(ns, func() GlobalFactory { + if ns != "" { + // Return the cluster wide factory if one exists. + if cw, ok := fm.Load(""); ok { + return cw + } + } k8sOpts, argoOpts := getOpts(ns) i := k8sapi.GetJoinedClientSetInterface(ctx) k8sFactory := informers.NewSharedInformerFactoryWithOptions(i, 0, k8sOpts...) From 87856f5fbba90fc2417f40798e06279ef5860508 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Fri, 10 Jan 2025 00:25:33 +0100 Subject: [PATCH 16/61] Fix problem with exclude-suffix being hidden by DNS search path. In some situations, a name ending with an exclude-suffix like "xyz.com" would be expanded by a search path into "xyz.com." and therefore not be excluded. Instead, the name was sent to the cluster to be resolved, causing an unnecessary load on its DNS server. Signed-off-by: Thomas Hallgren --- CHANGELOG.yml | 6 +++ docs/release-notes.md | 38 ++++++++++++++++ docs/release-notes.mdx | 26 +++++++++++ integration_test/kubeconfig_extension_test.go | 16 +++---- pkg/client/rootd/dns/server.go | 45 ++++++++++++++----- 5 files changed, 111 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.yml b/CHANGELOG.yml index 0aebb27d58..c920b92779 100644 --- a/CHANGELOG.yml +++ b/CHANGELOG.yml @@ -80,6 +80,12 @@ items: body: >- Using `telepresence list --namespace with a namespace different from the one that telepresence was connected to, would cause a deadlock, and then produce an empty list. + - type: bugfix + title: Fix problem with exclude-suffix being hidden by DNS search path. + body: >- + In some situations, a name ending with an exclude-suffix like "xyz.com" would be expanded by a search path + into "xyz.com." and therefore not be excluded. Instead, the name was sent to the cluster + to be resolved, causing an unnecessary load on its DNS server. - version: 2.21.3 date: 2025-02-06 notes: diff --git a/docs/release-notes.md b/docs/release-notes.md index 0c08047d8f..d525221649 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -62,6 +62,44 @@ macOS based systems will often PTR queries using nameslike `b._dns-sd._udp`, lb. Using `telepresence list --namespace with a namespace different from the one that telepresence was connected to, would cause a deadlock, and then produce an empty list.
+##
bugfix
Fix problem with exclude-suffix being hidden by DNS search path.
+
+ +In some situations, a name ending with an exclude-suffix like "xyz.com" would be expanded by a search path into "xyz.com." and therefore not be excluded. Instead, the name was sent to the cluster to be resolved, causing an unnecessary load on its DNS server. +
+ +## Version 2.21.3 (February 6) +##
bugfix
Using the --proxy-via flag would sometimes cause connection timeouts.
+
+ +Typically, a `telepresence connect --proxy-via =` would fail with a "deadline exceeded" message when several workloads were present in the namespace, the one targeted by the proxy-via didn't yet have an agent installed, and other workloads had an agent. This was due to a race condition in the logic for the agent-based port-forwards in the root daemon. The conditions causing this race are now eliminated. +
+ +##
bugfix
Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS.
+
+ +A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the TUN-device on macOS based clients. +
+ +##
bugfix
Ensure that annotation enabled traffic-agents are uninstall when uninstalling the traffic-manager.
+
+ +A traffic-agent injected because the workload had the inject annotation enabled would sometimes not get uninstalled when the traffic-manager was uninstalled. +
+ +## Version 2.21.2 (January 26) +##
bugfix
Fix panic when agentpf.client creates a Tunnel
+
+ +A race could occur where several requests where made to `agentpf.client.Tunnel` on a client that had errored when creating its port-forward to the agent. The implementation could handle one such requests but not several, resulting in a panic in situations where multiple simultaneous requests were made to the same client during a very short time period, +
+ +##
bugfix
Fix goroutine leak in dialer.
+
+ +The context passed to the `Tunnel` call that creates a stream for a dialer, was not cancelled when the dialer was finished, so the stream was never properly closed, leading to one dormant goroutine for each stream. +
+ ## Version 2.21.1 (December 17) ##
bugfix
[Allow ingest of serverless deployments without specifying an inject-container-ports annotation](https://github.com/telepresenceio/telepresence/issues/3741)
diff --git a/docs/release-notes.mdx b/docs/release-notes.mdx index 7f196dbffc..4d8de5dc0a 100644 --- a/docs/release-notes.mdx +++ b/docs/release-notes.mdx @@ -54,6 +54,32 @@ traffic to the original application container. This simplification offers severa Using the --namespace option with telepresence causes a deadlock. Using `telepresence list --namespace with a namespace different from the one that telepresence was connected to, would cause a deadlock, and then produce an empty list. + + Fix problem with exclude-suffix being hidden by DNS search path. + In some situations, a name ending with an exclude-suffix like "xyz.com" would be expanded by a search path into "xyz.com." and therefore not be excluded. Instead, the name was sent to the cluster to be resolved, causing an unnecessary load on its DNS server. + +## Version 2.21.3 (February 6) + + Using the --proxy-via flag would sometimes cause connection timeouts. + Typically, a `telepresence connect --proxy-via =` would fail with a "deadline exceeded" message when several workloads were present in the namespace, the one targeted by the proxy-via didn't yet have an agent installed, and other workloads had an agent. This was due to a race condition in the logic for the agent-based port-forwards in the root daemon. The conditions causing this race are now eliminated. + + + Fix panic in root daemon when using the "allow conflicting subnets" feature on macOS. + A regression was introduced in version 2.21.0, causing a panic due to an unimplemented method in the TUN-device on macOS based clients. + + + Ensure that annotation enabled traffic-agents are uninstall when uninstalling the traffic-manager. + A traffic-agent injected because the workload had the inject annotation enabled would sometimes not get uninstalled when the traffic-manager was uninstalled. + +## Version 2.21.2 (January 26) + + Fix panic when agentpf.client creates a Tunnel + A race could occur where several requests where made to `agentpf.client.Tunnel` on a client that had errored when creating its port-forward to the agent. The implementation could handle one such requests but not several, resulting in a panic in situations where multiple simultaneous requests were made to the same client during a very short time period, + + + Fix goroutine leak in dialer. + The context passed to the `Tunnel` call that creates a stream for a dialer, was not cancelled when the dialer was finished, so the stream was never properly closed, leading to one dormant goroutine for each stream. + ## Version 2.21.1 (December 17) Allow ingest of serverless deployments without specifying an inject-container-ports annotation diff --git a/integration_test/kubeconfig_extension_test.go b/integration_test/kubeconfig_extension_test.go index 0bf8d1c822..8458dc7c40 100644 --- a/integration_test/kubeconfig_extension_test.go +++ b/integration_test/kubeconfig_extension_test.go @@ -315,7 +315,7 @@ func (s *notConnectedSuite) Test_DNSSuffixRules() { defaults.IncludeSuffixes, defaults.ExcludeSuffixes, []string{ - `Lookup A "` + randomName + randomDomain, + `Lookup A "` + randomName + randomDomain + `."`, }, }, { @@ -325,8 +325,8 @@ func (s *notConnectedSuite) Test_DNSSuffixRules() { nil, nil, []string{ - `Cluster DNS included by include-suffix "` + randomDomain + `" for name "` + randomName + randomDomain, - `Lookup A "` + randomName + randomDomain, + `Cluster DNS included by include-suffix "` + randomDomain + `" for name "` + randomName + randomDomain + `"`, + `Lookup A "` + randomName + randomDomain + `."`, }, true, []string{randomDomain}, @@ -340,8 +340,8 @@ func (s *notConnectedSuite) Test_DNSSuffixRules() { nil, []string{randomDomain}, []string{ - `Cluster DNS included by include-suffix "` + randomDomain + `" for name "` + randomName + randomDomain, - `Lookup A "` + randomName + randomDomain, + `Cluster DNS included by include-suffix "` + randomDomain + `" for name "` + randomName + randomDomain + `"`, + `Lookup A "` + randomName + randomDomain + `."`, }, true, []string{randomDomain}, @@ -355,8 +355,8 @@ func (s *notConnectedSuite) Test_DNSSuffixRules() { nil, []string{randomDomain2}, []string{ - `Cluster DNS included by include-suffix "` + randomDomain + `" for name "` + randomName + randomDomain, - `Lookup A "` + randomName + randomDomain, + `Cluster DNS included by include-suffix "` + randomDomain + `" for name "` + randomName + randomDomain + `"`, + `Lookup A "` + randomName + randomDomain + `."`, }, true, []string{randomDomain}, @@ -456,7 +456,7 @@ func (s *notConnectedSuite) Test_DNSSuffixRules() { _, _ = net.DefaultResolver.LookupIPAddr(short, tt.domainName) // Give query time to reach telepresence and produce a log entry - dtime.SleepWithContext(ctx, 100*time.Millisecond) + dtime.SleepWithContext(ctx, 500*time.Millisecond) for _, wl := range tt.wantedLogEntry { _, err = rootLog.Seek(pos, io.SeekStart) diff --git a/pkg/client/rootd/dns/server.go b/pkg/client/rootd/dns/server.go index b969781274..0788cd779a 100644 --- a/pkg/client/rootd/dns/server.go +++ b/pkg/client/rootd/dns/server.go @@ -227,24 +227,36 @@ func (s *Server) shouldDoClusterLookup(query string) bool { // Skip configured exclude-suffixes unless also matched by an include-suffix // that is longer (i.e. more specific). - for _, es := range s.ExcludeSuffixes { - if strings.HasSuffix(name, es) { - // Exclude unless more specific include. - for _, is := range s.IncludeSuffixes { - if len(is) >= len(es) && strings.HasSuffix(name, is) { - dlog.Debugf(s.ctx, - "Cluster DNS included by include-suffix %q (overriding exclude-suffix %q) for name %q", is, es, name) - return true + suffixExcluded := func(n string) (included, excluded bool) { + for _, es := range s.ExcludeSuffixes { + if strings.HasSuffix(n, es) { + // Exclude unless more specific include. + for _, is := range s.IncludeSuffixes { + if len(is) >= len(es) && strings.HasSuffix(n, is) { + dlog.Debugf(s.ctx, + "Cluster DNS included by include-suffix %q (overriding exclude-suffix %q) for name %q", is, es, n) + return true, false + } } + dlog.Debugf(s.ctx, "Cluster DNS excluded by exclude-suffix %q for name %q", es, n) + return false, true } - dlog.Debugf(s.ctx, "Cluster DNS excluded by exclude-suffix %q for name %q", es, name) - return false } + return false, false + } + + if include, exclude := suffixExcluded(name); include || exclude { + return include } // Always include configured search paths + ln := len(name) for _, sfx := range s.search { - if strings.HasSuffix(name, sfx) { + li := ln - len(sfx) - 1 + if li > 0 && name[li] == '.' && strings.HasSuffix(name, sfx) { + if include, exclude := suffixExcluded(name[:li]); include || exclude { + return include + } dlog.Debugf(s.ctx, "Cluster DNS included by search %q of name %q", sfx, name) return true } @@ -252,7 +264,11 @@ func (s *Server) shouldDoClusterLookup(query string) bool { // Always include configured routes for sfx := range s.routes { - if strings.HasSuffix(name, sfx) { + li := ln - len(sfx) - 1 + if li > 0 && name[li] == '.' && strings.HasSuffix(name, sfx) { + if include, exclude := suffixExcluded(name[:li]); include || exclude { + return include + } dlog.Debugf(s.ctx, "Cluster DNS included by namespace %q of name %q", sfx, name) return true } @@ -267,6 +283,11 @@ func (s *Server) shouldDoClusterLookup(query string) bool { // Always include configured includeSuffixes for _, sfx := range s.IncludeSuffixes { if strings.HasSuffix(name, sfx) { + if sfx[0] == '.' { + if include, exclude := suffixExcluded(strings.TrimSuffix(name, sfx)); include || exclude { + return include + } + } dlog.Debugf(s.ctx, "Cluster DNS included by include-suffix %q for name %q", sfx, name) return true From b4c716d797fd5c5e85a607150b2031fb28837615 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Wed, 8 Jan 2025 18:32:42 +0100 Subject: [PATCH 17/61] New telepresence replace command. The new `telepresence replace` command simplifies and clarifies container replacement. Previously, the `--replace` flag within the `telepresence intercept` command was used to replace containers. However, this approach introduced inconsistencies and limitations: * **Confusion:** Using a flag to modify the core function of a command designed for traffic interception led to ambiguity. * **Inaccurate Behavior:** Replacement was not possible when no incoming traffic was intercepted, as the command's design focused on traffic routing. To address these issues, the `--replace` flag within `telepresence intercept` has been deprecated. The new `telepresence replace` command provides a dedicated and consistent method for replacing containers, enhancing clarity and reliability. Key differences between `replace` and `intercept`: 1. **Scope:** The `replace` command targets and affects an entire container, impacting all its traffic, while an `intercept` targets specific services and/or service/container ports. 2. **Port Declarations:** Remote ports specified using the `--port` flag are container ports. 3. **No Default Port:** A `replace` can occur without intercepting any ports. 4. **Container State:** During a `replace`, the original container is no longer active within the cluster. The deprecated `--replace` flag still works, but is hidden from the `telepresence intercept` command help, and will print a deprecation warning when used. Signed-off-by: Thomas Hallgren --- CHANGELOG.yml | 46 +- cmd/traffic/cmd/agent/agent.go | 2 +- cmd/traffic/cmd/agent/containerstate.go | 49 +- cmd/traffic/cmd/agent/intercepttarget.go | 12 +- cmd/traffic/cmd/agent/state.go | 35 +- cmd/traffic/cmd/agent/state_test.go | 8 +- cmd/traffic/cmd/manager/mutator/watcher.go | 18 +- cmd/traffic/cmd/manager/service.go | 35 +- cmd/traffic/cmd/manager/state/intercept.go | 185 ++- docs/release-notes.md | 52 +- docs/release-notes.mdx | 46 +- integration_test/workspace_watch_test.go | 1 + pkg/agentconfig/container.go | 3 + pkg/agentconfig/util.go | 25 - pkg/client/cli/cmd/intercept.go | 2 +- pkg/client/cli/cmd/leave.go | 33 +- pkg/client/cli/cmd/list.go | 60 +- pkg/client/cli/cmd/replace.go | 27 + pkg/client/cli/cmd/telepresence.go | 24 +- pkg/client/cli/ingest/info.go | 2 +- pkg/client/cli/intercept/command.go | 94 +- .../cli/intercept/describe_intercepts.go | 23 +- pkg/client/cli/intercept/info.go | 18 +- pkg/client/cli/intercept/state.go | 1 + pkg/client/userd/trafficmgr/intercept.go | 49 +- pkg/client/userd/trafficmgr/session.go | 33 +- rpc/connector/connector.pb.go | 716 ++++---- rpc/connector/connector.proto | 6 +- rpc/manager/manager.pb.go | 1447 +++++++++-------- rpc/manager/manager.proto | 3 + 30 files changed, 1774 insertions(+), 1281 deletions(-) create mode 100644 pkg/client/cli/cmd/replace.go diff --git a/CHANGELOG.yml b/CHANGELOG.yml index c920b92779..acc8104bd5 100644 --- a/CHANGELOG.yml +++ b/CHANGELOG.yml @@ -27,6 +27,42 @@ items: - version: 2.22.0 date: (TBD) notes: + - type: feature + title: New telepresence replace command. + body: |- + The new `telepresence replace` command simplifies and clarifies container replacement. + + Previously, the `--replace` flag within the `telepresence intercept` command was used to replace containers. + However, this approach introduced inconsistencies and limitations: + + * **Confusion:** Using a flag to modify the core function of a command designed for traffic interception led + to ambiguity. + * **Inaccurate Behavior:** Replacement was not possible when no incoming traffic was intercepted, as the + command's design focused on traffic routing. + + To address these issues, the `--replace` flag within `telepresence intercept` has been deprecated. The new + `telepresence replace` command provides a dedicated and consistent method for replacing containers, enhancing + clarity and reliability. + + Key differences between `replace` and `intercept`: + + 1. **Scope:** The `replace` command targets and affects an entire container, impacting all its traffic, while + an `intercept` targets specific services and/or service/container ports. + 2. **Port Declarations:** Remote ports specified using the `--port` flag are container ports. + 3. **No Default Port:** A `replace` can occur without intercepting any ports. + 4. **Container State:** During a `replace`, the original container is no longer active within the cluster. + + The deprecated `--replace` flag still works, but is hidden from the `telepresence intercept` command help, and + will print a deprecation warning when used. + - type: feature + title: No dormant container present during replace. + body: |- + Telepresence will no longer inject a dormant container during a `telepresence replace` operation. Instead, the + Traffic Agent now directly serves as the replacement container, eliminating the need to forward traffic to the + original application container. This simplification offers several advantages when using the `--replace` flag: + + - **Removal of the init-container:** The need for a separate init-container is no longer necessary. + - **Elimination of port renames:** Port renames within the intercepted pod are no longer required. - type: feature title: One single invocation of the Telepresence intercept command can now intercept multiple ports. body: >- @@ -49,16 +85,6 @@ items: values: `. ``` docs: install/manager#static-versus-dynamic-namespace-selection - - type: feature - title: Removal of the dormant container during intercept with --replace. - body: |- - During a `telepresence intercept --replace operation`, the previously injected dormant container has been - removed. The Traffic Agent now directly serves as the replacement container, eliminating the need to forward - traffic to the original application container. This simplification offers several advantages when using the - `--replace` flag: - - - **Removal of the init-container:** The need for a separate init-container is no longer necessary. - - **Elimination of port renames:** Port renames within the intercepted pod are no longer required. - type: change title: Drop deprecated current-cluster-id command. body: >- diff --git a/cmd/traffic/cmd/agent/agent.go b/cmd/traffic/cmd/agent/agent.go index 5e6b05046f..22a7431c51 100644 --- a/cmd/traffic/cmd/agent/agent.go +++ b/cmd/traffic/cmd/agent/agent.go @@ -158,7 +158,7 @@ func sidecar(ctx context.Context, s State, info *rpc.AgentInfo) error { ac := s.AgentConfig() for _, cn := range ac.Containers { ci := info.Containers[cn.Name] - s.AddContainerState(cn.Name, NewContainerState(ci.MountPoint, ci.Environment)) + s.AddContainerState(cn.Name, NewContainerState(s, cn, ci.MountPoint, ci.Environment)) // Group the container's intercepts by agent port icStates := make(map[agentconfig.PortAndProto][]*agentconfig.Intercept, len(cn.Intercepts)) diff --git a/cmd/traffic/cmd/agent/containerstate.go b/cmd/traffic/cmd/agent/containerstate.go index 519c205aeb..597945586a 100644 --- a/cmd/traffic/cmd/agent/containerstate.go +++ b/cmd/traffic/cmd/agent/containerstate.go @@ -1,21 +1,64 @@ package agent +import ( + "context" + + "github.com/datawire/dlib/dlog" + "github.com/telepresenceio/telepresence/rpc/v2/manager" + "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" +) + type containerState struct { + State + container *agentconfig.Container mountPoint string env map[string]string } -func (c containerState) MountPoint() string { +func (c *containerState) MountPoint() string { return c.mountPoint } -func (c containerState) Env() map[string]string { +func (c *containerState) Env() map[string]string { return c.env } +func (c *containerState) Name() string { + return c.container.Name +} + +func (c *containerState) Replace() bool { + return bool(c.container.Replace) +} + +// HandleIntercepts on the containerState takes care of intercepts that just replaces a container and do not declare +// any ports. Without port declarations, there will be no Intercept entries for an fwdState to handle. +func (c *containerState) HandleIntercepts(ctx context.Context, iis []*manager.InterceptInfo) (rs []*manager.ReviewInterceptRequest) { + for _, ii := range iis { + if ii.Disposition == manager.InterceptDispositionType_WAITING { + spec := ii.Spec + if c.Replace() && c.Name() == spec.ContainerName && spec.ContainerPort == 0 { + dlog.Debugf(ctx, "container %s handling replace %s", c.Name(), spec.Name) + rs = append(rs, &manager.ReviewInterceptRequest{ + Id: ii.Id, + Disposition: manager.InterceptDispositionType_ACTIVE, + PodIp: c.PodIP(), + SftpPort: int32(c.SftpPort()), + FtpPort: int32(c.FtpPort()), + MountPoint: c.MountPoint(), + Environment: c.Env(), + }) + } + } + } + return rs +} + // NewContainerState creates a ContainerState that provides the environment variables and the mount point for a container. -func NewContainerState(mountPoint string, env map[string]string) ContainerState { +func NewContainerState(s State, cn *agentconfig.Container, mountPoint string, env map[string]string) ContainerState { return &containerState{ + State: s, + container: cn, mountPoint: mountPoint, env: env, } diff --git a/cmd/traffic/cmd/agent/intercepttarget.go b/cmd/traffic/cmd/agent/intercepttarget.go index 208f6dcf67..612d57cf34 100644 --- a/cmd/traffic/cmd/agent/intercepttarget.go +++ b/cmd/traffic/cmd/agent/intercepttarget.go @@ -6,7 +6,7 @@ import ( "fmt" "strconv" - v1 "k8s.io/api/core/v1" + core "k8s.io/api/core/v1" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/rpc/v2/manager" @@ -39,9 +39,11 @@ func NewInterceptTarget(ics []*agentconfig.Intercept) InterceptTarget { } func (cp InterceptTarget) MatchForSpec(spec *manager.InterceptSpec) bool { - for _, ic := range cp { - if agentconfig.SpecMatchesIntercept(spec, ic) { - return true + if cnPort := uint16(spec.ContainerPort); cnPort > 0 { + for _, ic := range cp { + if cnPort == ic.ContainerPort && ic.Protocol == core.Protocol(spec.Protocol) { + return true + } } } return false @@ -59,7 +61,7 @@ func (cp InterceptTarget) ContainerPortName() string { return cp[0].ContainerPortName } -func (cp InterceptTarget) Protocol() v1.Protocol { +func (cp InterceptTarget) Protocol() core.Protocol { return cp[0].Protocol } diff --git a/cmd/traffic/cmd/agent/state.go b/cmd/traffic/cmd/agent/state.go index 6abc1ae259..9db5994e4c 100644 --- a/cmd/traffic/cmd/agent/state.go +++ b/cmd/traffic/cmd/agent/state.go @@ -37,6 +37,9 @@ type State interface { } type ContainerState interface { + State + Name() string + Replace() bool MountPoint() string Env() map[string]string } @@ -111,18 +114,38 @@ func (s *state) InterceptStates() []InterceptState { func (s *state) HandleIntercepts(ctx context.Context, iis []*manager.InterceptInfo) []*manager.ReviewInterceptRequest { var rs []*manager.ReviewInterceptRequest + + // Keep track of all InterceptInfos handled by interceptStates + handled := make([]bool, len(iis)) for _, ist := range s.interceptStates { ms := make([]*manager.InterceptInfo, 0, len(iis)) - for _, ii := range iis { - ic := ist.Target() - if ic.MatchForSpec(ii.Spec) { - dlog.Debugf(ctx, "intercept id %s svc=%q, portId=%q matches target protocol=%s, agentPort=%d, containerPort=%d", - ii.Id, ii.Spec.ServiceName, ii.Spec.PortIdentifier, ic.Protocol(), ic.AgentPort(), ic.ContainerPort()) - ms = append(ms, ii) + for i, ii := range iis { + if !handled[i] { + ic := ist.Target() + if ic.MatchForSpec(ii.Spec) { + dlog.Debugf(ctx, "intercept id %s svc=%q, portId=%q matches target protocol=%s, agentPort=%d, containerPort=%d", + ii.Id, ii.Spec.ServiceName, ii.Spec.PortIdentifier, ic.Protocol(), ic.AgentPort(), ic.ContainerPort()) + ms = append(ms, ii) + handled[i] = true + } } } rs = append(rs, ist.HandleIntercepts(ctx, ms)...) } + + // Collect InterceptInfos weren't handled by interceptStates + var unhandled []*manager.InterceptInfo + for i, ok := range handled { + if !ok { + unhandled = append(unhandled, iis[i]) + } + } + if len(unhandled) > 0 { + // Let containerStates handle the rest. + for _, cn := range s.containerStates { + rs = append(rs, cn.HandleIntercepts(ctx, unhandled)...) + } + } return rs } diff --git a/cmd/traffic/cmd/agent/state_test.go b/cmd/traffic/cmd/agent/state_test.go index 52f22c3cf7..1bf24fd975 100644 --- a/cmd/traffic/cmd/agent/state_test.go +++ b/cmd/traffic/cmd/agent/state_test.go @@ -40,7 +40,7 @@ func makeFS(t *testing.T, ctx context.Context) (forwarder.Interceptor, agent.Sta s := agent.NewState(c) cn := c.AgentConfig().Containers[0] cnMountPoint := filepath.Join(agentconfig.ExportsMountPoint, filepath.Base(cn.MountPoint)) - s.AddContainerState(cn.Name, agent.NewContainerState(cnMountPoint, map[string]string{})) + s.AddContainerState(cn.Name, agent.NewContainerState(s, cn, cnMountPoint, map[string]string{})) s.AddInterceptState(s.NewInterceptState(f, agent.NewInterceptTarget(cn.Intercepts), cn.Name)) return f, s } @@ -81,6 +81,8 @@ func TestState_HandleIntercepts(t *testing.T) { Namespace: namespace, ServiceName: serviceName, PortIdentifier: "http", + ContainerPort: 8080, + Protocol: string(core.ProtocolTCP), TargetPort: 8080, }, Id: "intercept-01", @@ -94,6 +96,8 @@ func TestState_HandleIntercepts(t *testing.T) { Namespace: namespace, ServiceName: serviceName, PortIdentifier: "http", + ContainerPort: 8080, + Protocol: string(core.ProtocolTCP), TargetPort: 8080, }, Id: "intercept-02", @@ -115,7 +119,7 @@ func TestState_HandleIntercepts(t *testing.T) { cepts[1].Disposition = rpc.InterceptDispositionType_WAITING reviews = s.HandleIntercepts(ctx, cepts) - a.Len(reviews, 2) + require.Len(t, reviews, 2) a.Equal("", f.InterceptId()) // Reviews are in the correct order diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index 9fbdf12ea1..924ee2da47 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -102,21 +102,18 @@ func (c *configWatcher) isRolloutNeeded(ctx context.Context, wl k8sapi.Workload, if wl.GetDeletionTimestamp() != nil { return false } - if ia, ok := podMeta.GetAnnotations()[agentconfig.InjectAnnotation]; ok { + + injectAnnotation, ok := podMeta.GetAnnotations()[agentconfig.InjectAnnotation] + if ok { // Annotation controls injection, so no explicit rollout is needed unless the deployment was added before the // traffic-manager or the traffic-manager already received an injection event but failed due to the lack // of an agent config. if c.running.Load() { if c.receivedPrematureInjectEvent(wl) { dlog.Debugf(ctx, "Rollout of %s.%s is necessary. Pod template has inject annotation %s and a premature injection event was received", - wl.GetName(), wl.GetNamespace(), ia) + wl.GetName(), wl.GetNamespace(), injectAnnotation) return true } - if wl.GetCreationTimestamp().After(c.startedAt) { - dlog.Debugf(ctx, "Rollout of %s.%s is not necessary. Pod template has inject annotation %s", - wl.GetName(), wl.GetNamespace(), ia) - return false - } } } podLabels := podMeta.GetLabels() @@ -158,6 +155,11 @@ func (c *configWatcher) isRolloutNeeded(ctx context.Context, wl k8sapi.Workload, } // Rollout if there are no running pods if runningPods == 0 { + if injectAnnotation != "" && wl.GetCreationTimestamp().After(c.startedAt) { + dlog.Debugf(ctx, "Rollout of %s.%s is not necessary. Pod template has inject annotation %s", + wl.GetName(), wl.GetNamespace(), injectAnnotation) + return false + } if ac != nil { dlog.Debugf(ctx, "Rollout of %s.%s is necessary. An agent is desired and there are no pods", wl.GetName(), wl.GetNamespace()) @@ -231,7 +233,7 @@ func isRolloutNeededForPod(ctx context.Context, ac *agentconfig.Sidecar, name, n name, namespace, cn.Name) } } else if found == nil { - return fmt.Sprintf("Rollout of %s.%s is necessary. The %s container should not be replaced", + return fmt.Sprintf("Rollout of %s.%s is necessary. The %s container must be restored", name, namespace, cn.Name) } } diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index ce0cd32c3f..d399d95ff9 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -415,9 +415,11 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(string, *rp return nil } agentSessionIDs := slices.Sorted(maps.Keys(snapshot.State)) - agents := make([]*rpc.AgentInfo, len(agentSessionIDs)) - for i, agentSessionID := range agentSessionIDs { - agents[i] = snapshot.State[agentSessionID] + agents := make([]*rpc.AgentInfo, 0, len(agentSessionIDs)) + for _, agentSessionID := range agentSessionIDs { + if as := s.state.GetSession(agentSessionID); as != nil && as.Active() { + agents = append(agents, snapshot.State[agentSessionID]) + } } if slices.EqualFunc(agents, lastSnap, infosEqual) { continue @@ -467,12 +469,16 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W } if agent := s.state.GetAgent(sessionID); agent != nil { - // sessionID refers to an agent session. Include everything for the agent, including pod-port children. filter = func(id string, info *rpc.InterceptInfo) bool { if info.Spec.Namespace != agent.Namespace || info.Spec.Agent != agent.Name { // Don't return intercepts for different agents. return false } + if as := s.state.GetSession(sessionID); as == nil || !as.Active() { + // Session is no longer active + return false + } + // Don't return intercepts that aren't in a "agent-owned" state. switch info.Disposition { case rpc.InterceptDispositionType_WAITING, @@ -508,6 +514,10 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W dlog.Debugf(ctx, "WatchIntercepts request cancelled") return nil } + if as := s.state.GetSession(sessionID); as == nil || !as.Active() { + dlog.Debugf(ctx, "WatchIntercepts session no longer active") + return nil + } dlog.Debugf(ctx, "WatchIntercepts sending update") intercepts := make([]*rpc.InterceptInfo, 0, len(snapshot.State)) for _, intercept := range snapshot.State { @@ -597,13 +607,6 @@ func (s *service) CreateIntercept(ctx context.Context, ciReq *rpc.CreateIntercep return nil, status.Error(codes.InvalidArgument, val) } - if ciReq.InterceptSpec.Replace { - _, err := s.state.PrepareIntercept(ctx, ciReq) - if err != nil { - return nil, err - } - } - client, interceptInfo, err := s.state.AddIntercept(ctx, ciReq) if err != nil { return nil, err @@ -684,17 +687,17 @@ func (s *service) ReviewIntercept(ctx context.Context, rIReq *rpc.ReviewIntercep sessionID := rIReq.GetSession().GetSessionId() ceptID := rIReq.Id + agent := s.state.GetActiveAgent(sessionID) + if agent == nil { + return &empty.Empty{}, nil + } + if rIReq.Disposition == rpc.InterceptDispositionType_AGENT_ERROR { dlog.Errorf(ctx, "ReviewIntercept called: %s - %s: %s", ceptID, rIReq.Disposition, rIReq.Message) } else { dlog.Debugf(ctx, "ReviewIntercept called: %s - %s", ceptID, rIReq.Disposition) } - agent := s.state.GetActiveAgent(sessionID) - if agent == nil { - return &empty.Empty{}, nil - } - s.removeExcludedEnvVars(rIReq.Environment) intercept := s.state.UpdateIntercept(ceptID, func(intercept *rpc.InterceptInfo) { diff --git a/cmd/traffic/cmd/manager/state/intercept.go b/cmd/traffic/cmd/manager/state/intercept.go index 625f717e4b..8c8776eb85 100644 --- a/cmd/traffic/cmd/manager/state/intercept.go +++ b/cmd/traffic/cmd/manager/state/intercept.go @@ -30,9 +30,30 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/errcat" + "github.com/telepresenceio/telepresence/v2/pkg/informer" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" ) +func (s *state) inactivateIfHasContainer(ctx context.Context, n, ns, cn string) { + api := informer.GetK8sFactory(ctx, ns).Core().V1().Pods().Lister().Pods(ns) + for sessionID, ai := range s.getAgentsByName(n, ns) { + pod, err := api.Get(ai.PodName) + if err != nil { + continue + } + cns := pod.Spec.Containers + for i := range cns { + if cns[i].Name == cn { + dlog.Debugf(ctx, "Inactivating pod %s %s because it contains container %q which is about to be replaced", pod.Name, pod.Status.PodIP, cn) + if as, ok := s.GetSession(sessionID).(*agentSessionState); ok { + as.active.Store(false) + } + break + } + } + } +} + // PrepareIntercept ensures that the given request can be matched against the intercept configuration of // the workload that it references. It returns a PreparedIntercept where all intercepted ports have been // qualified with a container port and if applicable, with service name and a service port name. @@ -57,10 +78,10 @@ func (s *state) PrepareIntercept( }() interceptError := func(err error) (*rpc.PreparedIntercept, error) { + dlog.Errorf(ctx, "PrepareIntercept error %v", err) if _, ok := status.FromError(err); ok { return nil, err } - dlog.Errorf(ctx, "PrepareIntercept error %v", err) return &rpc.PreparedIntercept{Error: err.Error(), ErrorCategory: int32(errcat.GetCategory(err))}, nil } @@ -74,38 +95,116 @@ func (s *state) PrepareIntercept( return interceptError(err) } + if spec.Replace { + // If we already have an agent config, then any pod that matches the spec and also has the container that we want + // to replace must be blacklisted now, or that agent will interfere with the replace. + mm := mutator.GetMap(ctx) + scx, err := mm.Get(ctx, spec.Agent, spec.Namespace) + if err != nil { + return interceptError(err) + } + if scx != nil { + if cn, err := findContainer(scx.AgentConfig(), spec); err == nil { + s.inactivateIfHasContainer(ctx, spec.Name, spec.Namespace, cn.Name) + } + } + } + ac, _, err := s.ensureAgent(ctx, wl, s.isExtended(spec), spec) if err != nil { return interceptError(err) } - cn, ic, err := findIntercept(ac, spec) + + pi = &rpc.PreparedIntercept{ + Namespace: ac.Namespace, + AgentImage: ac.AgentImage, + WorkloadKind: ac.WorkloadKind, + ContainerName: spec.ContainerName, + ServiceName: spec.ServiceName, + } + + var cn *agentconfig.Container + if spec.NoDefaultPort { + if cn, err = findContainer(ac, spec); err == nil { + pi.ContainerName = cn.Name + pi.ServiceName = "" + if spec.PortIdentifier == "all" { + prepareAllContainerPorts(cn, pi) + } else if spec.PortIdentifier != "" { + err = s.preparePorts(ac, cn, cr, pi) + } + } + } else { + err = s.preparePorts(ac, nil, cr, pi) + } + if err != nil { - return interceptError(err) + return interceptError(errcat.User.New(err)) } + return pi, nil +} + +func prepareAllContainerPorts(cn *agentconfig.Container, pi *rpc.PreparedIntercept) { + if ni := len(cn.Intercepts); ni > 0 { + // Put first port in the intercept itself + i0 := cn.Intercepts[0] + pi.ContainerPort = int32(i0.ContainerPort) + pi.Protocol = string(i0.Protocol) + if ni > 1 { + // Put remaining ports in PodPorts with a 1:1 mapping to target port on client. + pi.PodPorts = make([]string, ni-1) + for i := 1; i < ni; i++ { + ic := cn.Intercepts[i] + pi.PodPorts[i-1] = fmt.Sprintf("%d:%d/%s", ic.ContainerPort, ic.ContainerPort, ic.Protocol) + } + } + } +} + +func (s *state) preparePorts(ac *agentconfig.Sidecar, cn *agentconfig.Container, cr *rpc.CreateInterceptRequest, pi *rpc.PreparedIntercept) (err error) { + spec := cr.InterceptSpec + portID := agentconfig.PortIdentifier(spec.PortIdentifier) + containerOnly := cn != nil + + var ic *agentconfig.Intercept + if containerOnly { + ic, err = findContainerIntercept(ac, cn, portID) + } else { + cn, ic, err = findIntercept2(ac, pi.ServiceName, pi.ContainerName, portID) + } + if err != nil { + return err + } + uniqueContainerPorts := make(map[agentconfig.PortAndProto]struct{}) uniqueContainerPorts[agentconfig.PortAndProto{Proto: ic.Protocol, Port: ic.ContainerPort}] = struct{}{} var podPorts []string if len(spec.PodPorts) > 0 { - podPorts = make([]string, len(spec.PodPorts)) uniqueTargets := make(map[agentconfig.PortAndProto]struct{}) uniqueTargets[agentconfig.PortAndProto{Proto: ic.Protocol, Port: uint16(spec.TargetPort)}] = struct{}{} + podPorts = make([]string, len(spec.PodPorts)) for i, pms := range spec.PodPorts { pm := agentconfig.PortMapping(pms) - _, pmIc, err := findIntercept2(ac, spec.ServiceName, spec.ContainerName, pm.From()) + var pmIc *agentconfig.Intercept + if containerOnly { + pmIc, err = findContainerIntercept(ac, cn, pm.From()) + } else { + _, pmIc, err = findIntercept2(ac, spec.ServiceName, spec.ContainerName, pm.From()) + } if err != nil { - return interceptError(err) + return err } to := pm.To() if _, ok := uniqueTargets[to]; ok { - return interceptError(errcat.User.Newf("multiple port definitions targeting %s", to)) + return fmt.Errorf("multiple port definitions targeting %s", to) } uniqueTargets[to] = struct{}{} from := agentconfig.PortAndProto{Proto: pmIc.Protocol, Port: pmIc.ContainerPort} if _, ok := uniqueContainerPorts[from]; ok { - return interceptError(errcat.User.Newf("multiple port definitions using container port %s", from)) + return fmt.Errorf("multiple port definitions using container port %s", from) } uniqueContainerPorts[from] = struct{}{} @@ -133,24 +232,19 @@ func (s *state) PrepareIntercept( client = cps[3] } } - return interceptError(errcat.User.Newf("container port %d is already intercepted by %s, intercept %s", cp.Port, client, name)) + return fmt.Errorf("container port %d is already intercepted by %s, intercept %s", cp.Port, client, name) } } } } - - return &rpc.PreparedIntercept{ - Namespace: ac.Namespace, - ServiceUid: string(ic.ServiceUID), - ServicePortName: ic.ServicePortName, - ContainerName: cn.Name, - Protocol: string(ic.Protocol), - ContainerPort: int32(ic.ContainerPort), - ServicePort: int32(ic.ServicePort), - AgentImage: ac.AgentImage, - WorkloadKind: ac.WorkloadKind, - PodPorts: podPorts, - }, nil + pi.ContainerName = cn.Name + pi.ServiceUid = string(ic.ServiceUID) + pi.ServicePortName = ic.ServicePortName + pi.Protocol = string(ic.Protocol) + pi.ContainerPort = int32(ic.ContainerPort) + pi.ServicePort = int32(ic.ServicePort) + pi.PodPorts = podPorts + return nil } func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptRequest) (*rpc.ClientInfo, *rpc.InterceptInfo, error) { @@ -401,20 +495,24 @@ func (s *state) RestoreAppContainer(ctx context.Context, ii *rpc.InterceptInfo) if err != nil { return false, err } - cn, _, err := findIntercept(sce.AgentConfig(), spec) + var cn *agentconfig.Container + if spec.NoDefaultPort { + cn, err = findContainer(sce.AgentConfig(), spec) + } else { + cn, _, err = findIntercept(sce.AgentConfig(), spec) + } if !(err == nil && cn.Replace) { return false, nil } cn.Replace = false // The pods for this workload will be killed once the new updated sidecar - // reaches the configmap. We remove them now, so that they don't continue to + // reaches the configmap. We inactivate them now, so that they don't continue to // review intercepts. - for sessionID, ai := range s.getAgentsByName(n, ns) { + for sessionID := range s.getAgentsByName(n, ns) { if as, ok := s.GetSession(sessionID).(*agentSessionState); ok { as.active.Store(false) } - mm.Blacklist(ai.PodName, ns) } return updateSidecar(sce, cm, n) }) @@ -522,7 +620,12 @@ func (s *state) getOrCreateAgentConfig( ac := sce.AgentConfig() if spec != nil { - cn, _, err := findIntercept(ac, spec) + var cn *agentconfig.Container + if spec.NoDefaultPort { + cn, err = findContainer(ac, spec) + } else { + cn, _, err = findIntercept(ac, spec) + } if err != nil { return false, err } @@ -779,6 +882,24 @@ func unmarshalConfigMapEntry(y string, name, namespace string) (agentconfig.Side return scx, nil } +// findContainer finds the container configuration that matches the given InterceptSpec. +func findContainer(ac *agentconfig.Sidecar, spec *rpc.InterceptSpec) (foundCN *agentconfig.Container, err error) { + if spec.ContainerName == "" { + if len(ac.Containers) == 1 { + return ac.Containers[0], nil + } + return nil, errcat.User.Newf("%s %s.%s has more than one container", + ac.WorkloadKind, ac.WorkloadName, ac.Namespace) + } + for _, cn := range ac.Containers { + if spec.ContainerName == cn.Name { + return cn, nil + } + } + return nil, errcat.User.Newf("%s %s.%s has no container named %s", + ac.WorkloadKind, ac.WorkloadName, ac.Namespace, spec.ContainerName) +} + // findIntercept finds the intercept configuration that matches the given InterceptSpec's service/service port or container port. func findIntercept(ac *agentconfig.Sidecar, spec *rpc.InterceptSpec) (foundCN *agentconfig.Container, foundIC *agentconfig.Intercept, err error) { return findIntercept2(ac, spec.ServiceName, spec.ContainerName, agentconfig.PortIdentifier(spec.PortIdentifier)) @@ -854,6 +975,16 @@ func findIntercept2(ac *agentconfig.Sidecar, serviceName, containerName string, return nil, nil, errcat.User.Newf("%s %s.%s has no interceptable port%s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace, ss) } +// findContainerIntercept finds the intercept configuration that matches container port. +func findContainerIntercept(ac *agentconfig.Sidecar, cn *agentconfig.Container, pi agentconfig.PortIdentifier) (*agentconfig.Intercept, error) { + for _, ic := range cn.Intercepts { + if agentconfig.IsInterceptForContainer(pi, ic) { + return ic, nil + } + } + return nil, errcat.User.Newf("%s %s.%s has no container port matching %s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace, pi) +} + type InterceptFinalizer func(ctx context.Context, interceptInfo *rpc.InterceptInfo) error type interceptState struct { diff --git a/docs/release-notes.md b/docs/release-notes.md index d525221649..44e37643d2 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -2,6 +2,46 @@ [comment]: # (Code generated by relnotesgen. DO NOT EDIT.) # Telepresence Release Notes ## Version 2.22.0 +##
feature
New telepresence replace command.
+
+ +The new `telepresence replace` command simplifies and clarifies container replacement. + +Previously, the `--replace` flag within the `telepresence intercept` command was used to replace containers. +However, this approach introduced inconsistencies and limitations: + +* **Confusion:** Using a flag to modify the core function of a command designed for traffic interception led + to ambiguity. +* **Inaccurate Behavior:** Replacement was not possible when no incoming traffic was intercepted, as the + command's design focused on traffic routing. + +To address these issues, the `--replace` flag within `telepresence intercept` has been deprecated. The new +`telepresence replace` command provides a dedicated and consistent method for replacing containers, enhancing +clarity and reliability. + +Key differences between `replace` and `intercept`: + +1. **Scope:** The `replace` command targets and affects an entire container, impacting all its traffic, while + an `intercept` targets specific services and/or service/container ports. +2. **Port Declarations:** Remote ports specified using the `--port` flag are container ports. +3. **No Default Port:** A `replace` can occur without intercepting any ports. +4. **Container State:** During a `replace`, the original container is no longer active within the cluster. + +The deprecated `--replace` flag still works, but is hidden from the `telepresence intercept` command help, and +will print a deprecation warning when used. +
+ +##
feature
No dormant container present during replace.
+
+ +Telepresence will no longer inject a dormant container during a `telepresence replace` operation. Instead, the +Traffic Agent now directly serves as the replacement container, eliminating the need to forward traffic to the +original application container. This simplification offers several advantages when using the `--replace` flag: + +- **Removal of the init-container:** The need for a separate init-container is no longer necessary. +- **Elimination of port renames:** Port renames within the intercepted pod are no longer required. +
+ ##
feature
One single invocation of the Telepresence intercept command can now intercept multiple ports.
@@ -26,18 +66,6 @@ namespaceSelector: ```
-##
feature
Removal of the dormant container during intercept with --replace.
-
- -During a `telepresence intercept --replace operation`, the previously injected dormant container has been -removed. The Traffic Agent now directly serves as the replacement container, eliminating the need to forward -traffic to the original application container. This simplification offers several advantages when using the -`--replace` flag: - -- **Removal of the init-container:** The need for a separate init-container is no longer necessary. -- **Elimination of port renames:** Port renames within the intercepted pod are no longer required. -
- ##
change
Drop deprecated current-cluster-id command.
diff --git a/docs/release-notes.mdx b/docs/release-notes.mdx index 4d8de5dc0a..a1ac700c7a 100644 --- a/docs/release-notes.mdx +++ b/docs/release-notes.mdx @@ -8,6 +8,42 @@ import { Note, Title, Body } from '@site/src/components/ReleaseNotes' # Telepresence Release Notes ## Version 2.22.0 + + New telepresence replace command. + The new `telepresence replace` command simplifies and clarifies container replacement. + +Previously, the `--replace` flag within the `telepresence intercept` command was used to replace containers. +However, this approach introduced inconsistencies and limitations: + +* **Confusion:** Using a flag to modify the core function of a command designed for traffic interception led + to ambiguity. +* **Inaccurate Behavior:** Replacement was not possible when no incoming traffic was intercepted, as the + command's design focused on traffic routing. + +To address these issues, the `--replace` flag within `telepresence intercept` has been deprecated. The new +`telepresence replace` command provides a dedicated and consistent method for replacing containers, enhancing +clarity and reliability. + +Key differences between `replace` and `intercept`: + +1. **Scope:** The `replace` command targets and affects an entire container, impacting all its traffic, while + an `intercept` targets specific services and/or service/container ports. +2. **Port Declarations:** Remote ports specified using the `--port` flag are container ports. +3. **No Default Port:** A `replace` can occur without intercepting any ports. +4. **Container State:** During a `replace`, the original container is no longer active within the cluster. + +The deprecated `--replace` flag still works, but is hidden from the `telepresence intercept` command help, and +will print a deprecation warning when used. + + + No dormant container present during replace. + Telepresence will no longer inject a dormant container during a `telepresence replace` operation. Instead, the +Traffic Agent now directly serves as the replacement container, eliminating the need to forward traffic to the +original application container. This simplification offers several advantages when using the `--replace` flag: + +- **Removal of the init-container:** The need for a separate init-container is no longer necessary. +- **Elimination of port renames:** Port renames within the intercepted pod are no longer required. + One single invocation of the Telepresence intercept command can now intercept multiple ports. It is now possible to intercept multiple ports with one single invocation of `telepresence intercept` by just repeating the `--port` flag. @@ -28,16 +64,6 @@ namespaceSelector: values: `. ``` - - Removal of the dormant container during intercept with --replace. - During a `telepresence intercept --replace operation`, the previously injected dormant container has been -removed. The Traffic Agent now directly serves as the replacement container, eliminating the need to forward -traffic to the original application container. This simplification offers several advantages when using the -`--replace` flag: - -- **Removal of the init-container:** The need for a separate init-container is no longer necessary. -- **Elimination of port renames:** Port renames within the intercepted pod are no longer required. - Drop deprecated current-cluster-id command. The clusterID was deprecated some time ago, and replaced by the ID of the namespace where the traffic-manager is installed. diff --git a/integration_test/workspace_watch_test.go b/integration_test/workspace_watch_test.go index 7c0b36b6c3..4cbe2a99ab 100644 --- a/integration_test/workspace_watch_test.go +++ b/integration_test/workspace_watch_test.go @@ -44,6 +44,7 @@ func (s *notConnectedSuite) Test_WorkspaceListener() { spec.ServicePortName = pi.ServicePortName spec.ServiceUid = pi.ServiceUid spec.ContainerPort = pi.ContainerPort + spec.Protocol = pi.Protocol spec.ContainerName = pi.ContainerName if pi.ServiceUid != "" { if pi.ServicePortName != "" { diff --git a/pkg/agentconfig/container.go b/pkg/agentconfig/container.go index 3874a987af..1820e7d99e 100644 --- a/pkg/agentconfig/container.go +++ b/pkg/agentconfig/container.go @@ -149,6 +149,9 @@ func AgentContainer( } }) + if len(ports) == 0 { + ports = nil + } ac := &core.Container{ Name: ContainerName, Image: config.AgentImage, diff --git a/pkg/agentconfig/util.go b/pkg/agentconfig/util.go index 4ec9e79d28..81c7cad13f 100644 --- a/pkg/agentconfig/util.go +++ b/pkg/agentconfig/util.go @@ -1,30 +1,5 @@ package agentconfig -import ( - core "k8s.io/api/core/v1" - - "github.com/telepresenceio/telepresence/rpc/v2/manager" -) - -// SpecMatchesIntercept answers the question if an InterceptSpec matches the given -// Intercept config. The spec matches if: -// - its ServiceName is equal to the config's ServiceName -// - its PortIdentifier is equal to the config's ServicePortName, or can -// be parsed to an integer equal to the config's ServicePort -func SpecMatchesIntercept(spec *manager.InterceptSpec, ic *Intercept) bool { - if spec.ServiceName != "" && spec.ServiceName != ic.ServiceName { - return false - } - if spec.PortIdentifier != "" { - pi := PortIdentifier(spec.PortIdentifier) - if spec.ServiceUid != "" { - return IsInterceptForService(pi, ic) - } - return IsInterceptForContainer(pi, ic) - } - return uint16(spec.ContainerPort) == ic.ContainerPort && (spec.Protocol == "" || core.Protocol(spec.Protocol) == ic.Protocol) -} - // IsInterceptForService returns true when the given PortIdentifier is equal to the // config's ServicePortName, or can be parsed to an integer equal to the config's ServicePort. func IsInterceptForService(pi PortIdentifier, ic *Intercept) bool { diff --git a/pkg/client/cli/cmd/intercept.go b/pkg/client/cli/cmd/intercept.go index beea44204a..ef09c3d2a6 100644 --- a/pkg/client/cli/cmd/intercept.go +++ b/pkg/client/cli/cmd/intercept.go @@ -22,6 +22,6 @@ func interceptCmd() *cobra.Command { RunE: ic.Run, ValidArgsFunction: intercept.ValidArgs, } - ic.AddFlags(cmd) + ic.AddInterceptFlags(cmd) return cmd } diff --git a/pkg/client/cli/cmd/leave.go b/pkg/client/cli/cmd/leave.go index 59b94c49cb..7608d3d966 100644 --- a/pkg/client/cli/cmd/leave.go +++ b/pkg/client/cli/cmd/leave.go @@ -34,7 +34,7 @@ func leave() *cobra.Command { if err := connect.InitCommand(cmd); err != nil { return err } - return removeIngestOrIntercept(cmd.Context(), strings.TrimSpace(args[0]), containerName) + return disengage(cmd.Context(), strings.TrimSpace(args[0]), containerName) }, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { shellCompDir := cobra.ShellCompDirectiveNoFileComp @@ -46,7 +46,9 @@ func leave() *cobra.Command { } ctx := cmd.Context() userD := daemon.GetUserClient(ctx) - resp, err := userD.List(ctx, &connector.ListRequest{Filter: connector.ListRequest_INTERCEPTS | connector.ListRequest_INGESTS}) + resp, err := userD.List(ctx, &connector.ListRequest{ + Filter: connector.ListRequest_INTERCEPTS | connector.ListRequest_REPLACEMENTS | connector.ListRequest_INGESTS, + }) if err != nil { return nil, shellCompDir | cobra.ShellCompDirectiveError } @@ -72,22 +74,24 @@ func leave() *cobra.Command { return completions, shellCompDir }, } - cmd.Flags().StringVarP(&containerName, "container", "c", "", "Container name (only relevant for ingest)") + cmd.Flags().StringVarP(&containerName, "container", "c", "", "Container name") return cmd } -func removeIngestOrIntercept(ctx context.Context, name, container string) error { +func disengage(ctx context.Context, name, container string) error { userD := daemon.GetUserClient(ctx) var ic *manager.InterceptInfo var ig *connector.IngestInfo var env map[string]string var err error - if container == "" { - ic, err = userD.GetIntercept(ctx, &manager.GetInterceptRequest{Name: name}) - if err != nil && status.Code(err) != codes.NotFound { - return err - } + icName := name + if container != "" { + icName += "/" + container + } + ic, err = userD.GetIntercept(ctx, &manager.GetInterceptRequest{Name: icName}) + if err != nil && status.Code(err) != codes.NotFound { + return err } if ic == nil { @@ -99,8 +103,13 @@ func removeIngestOrIntercept(ctx context.Context, name, container string) error if status.Code(err) != codes.NotFound { return err } - // User probably misspelled the name of the intercept/ingest - return errcat.User.Newf("Intercept or ingest named %q not found", name) + + // User probably misspelled the name of the replace/intercept/ingest + msg := fmt.Sprintf("Found no replace, intercept, or ingest named %q", name) + if container != "" { + msg = fmt.Sprintf("%s with container %q", msg, container) + } + return errcat.User.New(msg) } env = ig.Environment } else { @@ -118,7 +127,7 @@ func removeIngestOrIntercept(ctx context.Context, name, container string) error } if ic != nil { - err = intercept.Result(userD.RemoveIntercept(ctx, &manager.RemoveInterceptRequest2{Name: name})) + err = intercept.Result(userD.RemoveIntercept(ctx, &manager.RemoveInterceptRequest2{Name: ic.Spec.Name})) } else if ig != nil { _, err = userD.LeaveIngest(ctx, &connector.IngestIdentifier{ WorkloadName: ig.Workload, diff --git a/pkg/client/cli/cmd/list.go b/pkg/client/cli/cmd/list.go index 332d81d2c3..c5dc3f2ea1 100644 --- a/pkg/client/cli/cmd/list.go +++ b/pkg/client/cli/cmd/list.go @@ -19,14 +19,18 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/ioutil" ) +const ( + includeIntercepts = iota + includeIngests + includeReplacements +) + type listCommand struct { - onlyIntercepts bool - onlyIngests bool - onlyAgents bool - onlyInterceptable bool - debug bool - namespace string - watch bool + inclusions [3]bool + onlyAgents bool + debug bool + namespace string + watch bool } func list() *cobra.Command { @@ -43,13 +47,18 @@ func list() *cobra.Command { ValidArgsFunction: cobra.NoFileCompletions, } flags := cmd.Flags() - flags.BoolVarP(&s.onlyIntercepts, "intercepts", "i", false, "intercepts only") - flags.BoolVarP(&s.onlyIngests, "ingests", "g", false, "ingests only") + flags.BoolVarP(&s.inclusions[includeIntercepts], "intercepts", "i", false, "intercepts") + flags.BoolVarP(&s.inclusions[includeIngests], "ingests", "g", false, "ingests") + flags.BoolVarP(&s.inclusions[includeReplacements], "replacements", "r", false, "replacements") flags.BoolVarP(&s.onlyAgents, "agents", "a", false, "with installed agents only") - flags.BoolVarP(&s.onlyInterceptable, "only-interceptable", "o", true, "interceptable workloads only") flags.BoolVar(&s.debug, "debug", false, "include debugging information") flags.StringVarP(&s.namespace, "namespace", "n", "", "If present, the namespace scope for this CLI request") + flags.BoolP("only-interceptable", "o", false, "") + of := flags.Lookup("only-interceptable") + of.Hidden = true + of.Deprecated = "Redundant since all workloads are eligible for ingest, intercept, or replace" + flags.BoolVarP(&s.watch, "watch", "w", false, "watch a namespace. --agents and --intercepts are disabled if this flag is set") wf := flags.Lookup("watch") wf.Hidden = true @@ -90,18 +99,21 @@ func (s *listCommand) list(cmd *cobra.Command, _ []string) error { stdout := cmd.OutOrStdout() ctx := cmd.Context() userD := daemon.GetUserClient(ctx) - var filter connector.ListRequest_Filter - switch { - case s.onlyIntercepts: - filter = connector.ListRequest_INTERCEPTS - case s.onlyIngests: - filter = connector.ListRequest_INGESTS - case s.onlyAgents: + filter := connector.ListRequest_UNSPECIFIED + for i := range s.inclusions { + if s.inclusions[i] { + switch i { + case includeIntercepts: + filter |= connector.ListRequest_INTERCEPTS + case includeReplacements: + filter |= connector.ListRequest_REPLACEMENTS + case includeIngests: + filter |= connector.ListRequest_INGESTS + } + } + } + if filter == connector.ListRequest_UNSPECIFIED && s.onlyAgents { filter = connector.ListRequest_INSTALLED_AGENTS - case s.onlyInterceptable: - filter = connector.ListRequest_INTERCEPTABLE - default: - filter = connector.ListRequest_EVERYTHING } cfg := client.GetConfig(ctx) @@ -176,12 +188,12 @@ func (s *listCommand) printList(ctx context.Context, workloads []*connector.Work return "progressing..." } if workload.AgentVersion != "" { - return "ready to intercept (traffic-agent already installed)" + return "ready to engage (traffic-agent already installed)" } if workload.NotInterceptableReason != "" { - return "not interceptable (traffic-agent not installed): " + workload.NotInterceptableReason + return "unable to engage (traffic-agent not installed): " + workload.NotInterceptableReason } else { - return "ready to intercept (traffic-agent not yet installed)" + return "ready to engage (traffic-agent not yet installed)" } } diff --git a/pkg/client/cli/cmd/replace.go b/pkg/client/cli/cmd/replace.go new file mode 100644 index 0000000000..c75ac54de4 --- /dev/null +++ b/pkg/client/cli/cmd/replace.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "github.com/spf13/cobra" + + "github.com/telepresenceio/telepresence/v2/pkg/client/cli/ann" + "github.com/telepresenceio/telepresence/v2/pkg/client/cli/intercept" +) + +func replaceCmd() *cobra.Command { + ic := &intercept.Command{} + cmd := &cobra.Command{ + Use: "replace [flags] [-- ]", + Args: cobra.MinimumNArgs(1), + Short: "Replace a container", + Annotations: map[string]string{ + ann.Session: ann.Required, + ann.UpdateCheckFormat: ann.Tel2, + }, + SilenceUsage: true, + SilenceErrors: true, + RunE: ic.RunReplace, + ValidArgsFunction: intercept.ValidArgs, + } + ic.AddReplaceFlags(cmd) + return cmd +} diff --git a/pkg/client/cli/cmd/telepresence.go b/pkg/client/cli/cmd/telepresence.go index a7172e56d7..e2d94131a3 100644 --- a/pkg/client/cli/cmd/telepresence.go +++ b/pkg/client/cli/cmd/telepresence.go @@ -135,10 +135,26 @@ func OnlySubcommands(cmd *cobra.Command, args []string) error { func WithSubCommands(ctx context.Context) context.Context { return MergeSubCommands(ctx, - configCmd(), connectCmd(), gatherLogs(), genYAML(), helmCmd(), - ingestCmd(), interceptCmd(), kubeauthCmd(), leave(), list(), listContexts(), listNamespaces(), loglevel(), quit(), statusCmd(), - dockerRunCmd(), curlCmd(), - uninstall(), version(), listNamespaces(), listContexts(), + configCmd(), + connectCmd(), + curlCmd(), + dockerRunCmd(), + gatherLogs(), + genYAML(), + helmCmd(), + ingestCmd(), + interceptCmd(), + kubeauthCmd(), + leave(), + list(), + listContexts(), + listNamespaces(), + loglevel(), + quit(), + replaceCmd(), + statusCmd(), + uninstall(), + version(), ) } diff --git a/pkg/client/cli/ingest/info.go b/pkg/client/cli/ingest/info.go index 28e9d8e376..b1ce71cd17 100644 --- a/pkg/client/cli/ingest/info.go +++ b/pkg/client/cli/ingest/info.go @@ -36,7 +36,7 @@ func NewInfo(ctx context.Context, ii *rpc.IngestInfo, mountError error) *Info { func (ii *Info) WriteTo(w io.Writer) (int64, error) { kvf := ioutil.DefaultKeyValueFormatter() kvf.Prefix = " " - kvf.Add("Container", ii.Container) + kvf.Add("Container name", ii.Container) kvf.Add("Workload kind", ii.WorkloadKind) if m := ii.Mount; m != nil { if m.LocalDir != "" { diff --git a/pkg/client/cli/intercept/command.go b/pkg/client/cli/intercept/command.go index 444335494f..411978e102 100644 --- a/pkg/client/cli/intercept/command.go +++ b/pkg/client/cli/intercept/command.go @@ -50,9 +50,10 @@ type Command struct { FormattedOutput bool DetailedOutput bool Silent bool + NoDefaultPort bool } -func (c *Command) AddFlags(cmd *cobra.Command) { +func (c *Command) AddInterceptFlags(cmd *cobra.Command) { flagSet := cmd.Flags() flagSet.StringVarP(&c.AgentName, "workload", "w", "", "Name of workload (Deployment, ReplicaSet, StatefulSet, Rollout) to intercept, if different from ") flagSet.StringSliceVarP(&c.Ports, "port", "p", nil, ``+ @@ -66,13 +67,13 @@ func (c *Command) AddFlags(cmd *cobra.Command) { `e.g. '--address 10.0.0.2'`, ) - flagSet.StringVar(&c.ServiceName, "service", "", "Name of service to intercept. If not provided, we will try to auto-detect one") + flagSet.StringVar(&c.ServiceName, "service", "", "Optional name of service to intercept. Sometimes needed to uniquely identify the intercepted port.") flagSet.StringVar(&c.ContainerName, "container", "", - "Name of container that provides the environment and mounts for the intercept. Defaults to the container matching the targetPort") + "Name of container that provides the environment and mounts for the intercept. Defaults to the container matching the first intercepted port.") flagSet.StringSliceVar(&c.ToPod, "to-pod", []string{}, ``+ - `An additional port to forward to the intercepted pod, will available for connections to localhost:PORT. `+ + `Additional ports to forward to the intercepted pod, will available for connections to localhost:PORT. `+ `Use this to, for example, access proxy/helper sidecars in the intercepted pod. The default protocol is TCP. `+ `Use /UDP for UDP ports`) @@ -90,11 +91,43 @@ func (c *Command) AddFlags(cmd *cobra.Command) { flagSet.BoolVarP(&c.Replace, "replace", "", false, `Indicates if the traffic-agent should replace application containers in workload pods. `+ `The default behavior is for the agent sidecar to be installed alongside existing containers.`) + flagSet.Lookup("replace").Deprecated = "Use the replace command." _ = cmd.RegisterFlagCompletionFunc("container", ingest.AutocompleteContainer) _ = cmd.RegisterFlagCompletionFunc("service", autocompleteService) } +func (c *Command) AddReplaceFlags(cmd *cobra.Command) { + flagSet := cmd.Flags() + flagSet.StringSliceVarP(&c.Ports, "port", "p", []string{"all"}, ``+ + `Local ports to forward to. Use : to uniquely identify container ports, where the is the port name or number. `+ + `Use "all" (the default) to forward all ports declared in the replaced container to their corresponding local port. `, + ) + + flagSet.StringVar(&c.Address, "address", "127.0.0.1", ``+ + `Local address to forward to, Only accepts IP address as a value, e.g. '--address 10.0.0.2'`, + ) + + flagSet.StringVar(&c.ContainerName, "container", "", + "Name of container that should be replaced. Can be omitted if the workload only has one container.") + + flagSet.StringSliceVar(&c.ToPod, "to-pod", []string{}, ``+ + `Additional ports to forward to the pod containing the replaced container, will available for connections to localhost:PORT. `+ + `Use this to, for example, access proxy/helper sidecars in the pod. The default protocol is TCP. `+ + `Use /UDP for UDP ports`) + + c.EnvFlags.AddFlags(flagSet) + c.MountFlags.AddFlags(flagSet, false) + c.DockerFlags.AddFlags(flagSet, "replaced") + + flagSet.StringVar(&c.WaitMessage, "wait-message", "", "Message to print when replace handler has started") + + flagSet.BoolVar(&c.DetailedOutput, "detailed-output", false, + `Provide very detailed info about the replace when used together with --output=json or --output=yaml'`) + + _ = cmd.RegisterFlagCompletionFunc("container", ingest.AutocompleteContainer) +} + func (c *Command) Validate(cmd *cobra.Command, positional []string) error { if len(positional) > 1 && cmd.Flags().ArgsLenAtDash() != 1 { return errcat.User.New("commands to be run with intercept must come after options") @@ -122,10 +155,54 @@ func (c *Command) Validate(cmd *cobra.Command, positional []string) error { return c.DockerFlags.Validate(c.Cmdline) } -func (c *Command) Run(cmd *cobra.Command, positional []string) error { - if err := c.Validate(cmd, positional); err != nil { +func (c *Command) ValidateReplace(cmd *cobra.Command, positional []string) error { + if len(positional) > 1 && cmd.Flags().ArgsLenAtDash() != 1 { + return errcat.User.New("commands to be run with replace must come after options") + } + c.Name = positional[0] + c.AgentName = c.Name + c.Cmdline = positional[1:] + c.FormattedOutput = output.WantsFormatted(cmd) + c.Mechanism = "tcp" + c.Replace = true + c.NoDefaultPort = true + + if c.ContainerName != "" { + c.Name += "/" + c.ContainerName + } + + if err := c.MountFlags.Validate(cmd); err != nil { return err } + if c.DockerFlags.Mount != "" && !c.MountFlags.Enabled { + return errors.New("--docker-mount cannot be used with --mount=false") + } + for i := range c.Ports { + if c.Ports[i] == "all" { + // Local port is unset, remote port is "all" + c.Ports[i] = ":all" + } + } + return c.DockerFlags.Validate(c.Cmdline) +} + +func (c *Command) Run(cmd *cobra.Command, positional []string) error { + err := c.Validate(cmd, positional) + if err == nil { + err = c.validatedRun(cmd) + } + return err +} + +func (c *Command) RunReplace(cmd *cobra.Command, positional []string) error { + err := c.ValidateReplace(cmd, positional) + if err == nil { + err = c.validatedRun(cmd) + } + return err +} + +func (c *Command) validatedRun(cmd *cobra.Command) error { if err := connect.InitCommand(cmd); err != nil { return err } @@ -199,12 +276,9 @@ func ValidArgs(cmd *cobra.Command, args []string, toComplete string) ([]string, if err := connect.InitCommand(cmd); err != nil { return nil, cobra.ShellCompDirectiveError } - req := connector.ListRequest{ - Filter: connector.ListRequest_INTERCEPTABLE, - } ctx := cmd.Context() - r, err := daemon.GetUserClient(ctx).List(ctx, &req) + r, err := daemon.GetUserClient(ctx).List(ctx, &connector.ListRequest{Filter: connector.ListRequest_UNSPECIFIED}) if err != nil { dlog.Debugf(ctx, "unable to get list of interceptable workloads: %v", err) return nil, cobra.ShellCompDirectiveError diff --git a/pkg/client/cli/intercept/describe_intercepts.go b/pkg/client/cli/intercept/describe_intercepts.go index 2b7bcd0a3c..907f7e8dae 100644 --- a/pkg/client/cli/intercept/describe_intercepts.go +++ b/pkg/client/cli/intercept/describe_intercepts.go @@ -12,10 +12,27 @@ import ( func DescribeIntercepts(ctx context.Context, iis []*manager.InterceptInfo, igs []*rpc.IngestInfo, volumeMountsPrevented error, debug bool) string { sb := strings.Builder{} if len(iis) > 0 { - sb.WriteString("intercepted") + var nis, ris []*manager.InterceptInfo for _, ii := range iis { - sb.WriteByte('\n') - describeIntercept(ctx, ii, volumeMountsPrevented, debug, &sb) + if ii.Spec.NoDefaultPort { + ris = append(ris, ii) + } else { + nis = append(nis, ii) + } + } + if len(nis) > 0 { + sb.WriteString("intercepted") + for _, ii := range nis { + sb.WriteByte('\n') + describeIntercept(ctx, ii, volumeMountsPrevented, debug, &sb) + } + } + if len(ris) > 0 { + sb.WriteString("replaced") + for _, ii := range ris { + sb.WriteByte('\n') + describeIntercept(ctx, ii, volumeMountsPrevented, debug, &sb) + } } } if len(igs) > 0 { diff --git a/pkg/client/cli/intercept/info.go b/pkg/client/cli/intercept/info.go index 866a78f521..a07a209c21 100644 --- a/pkg/client/cli/intercept/info.go +++ b/pkg/client/cli/intercept/info.go @@ -32,6 +32,7 @@ type Info struct { ServiceUID string `json:"service_uid,omitempty" yaml:"service_uid,omitempty"` ServicePortID string `json:"service_port_id,omitempty" yaml:"service_port_id,omitempty"` // ServicePortID is deprecated. Use PortID PortID string `json:"port_id,omitempty" yaml:"port_id,omitempty"` + ContainerName string `json:"container_name,omitempty" yaml:"container_name,omitempty"` ContainerPort int32 `json:"container_port,omitempty" yaml:"container_port,omitempty"` Protocol string `json:"protocol,omitempty" yaml:"protocol,omitempty"` Environment map[string]string `json:"environment,omitempty" yaml:"environment,omitempty"` @@ -40,6 +41,7 @@ type Info struct { Metadata map[string]string `json:"metadata,omitempty" yaml:"metadata,omitempty"` HttpFilter []string `json:"http_filter,omitempty" yaml:"http_filter,omitempty"` Global bool `json:"global,omitempty" yaml:"global,omitempty"` + Replace bool `json:"replace,omitempty" yaml:"replace,omitempty"` PreviewURL string `json:"preview_url,omitempty" yaml:"preview_url,omitempty"` Ingress *Ingress `json:"ingress,omitempty" yaml:"ingress,omitempty"` PodIP string `json:"pod_ip,omitempty" yaml:"pod_ip,omitempty"` @@ -89,6 +91,7 @@ func NewInfo(ctx context.Context, ii *manager.InterceptInfo, ro bool, mountError Mount: m, ServiceUID: spec.ServiceUid, PortID: spec.PortIdentifier, + ContainerName: spec.ContainerName, ContainerPort: spec.ContainerPort, Protocol: spec.Protocol, PodIP: ii.PodIp, @@ -97,6 +100,7 @@ func NewInfo(ctx context.Context, ii *manager.InterceptInfo, ro bool, mountError Metadata: ii.Metadata, HttpFilter: spec.MechanismArgs, Global: spec.Mechanism == "tcp", + Replace: spec.NoDefaultPort, // spec.Replace can't be used because it's set by deprecated --replace flag PreviewURL: PreviewURL(ii.PreviewDomain), Ingress: NewIngress(ii.PreviewSpec), } @@ -110,7 +114,11 @@ func NewInfo(ctx context.Context, ii *manager.InterceptInfo, ro bool, mountError func (ii *Info) WriteTo(w io.Writer) (int64, error) { kvf := ioutil.DefaultKeyValueFormatter() kvf.Prefix = " " - kvf.Add("Intercept name", ii.Name) + if ii.Replace { + kvf.Add("Container name", ii.ContainerName) + } else { + kvf.Add("Intercept name", ii.Name) + } kvf.Add("State", func() string { msg := "" if manager.InterceptDispositionType_value[ii.Disposition] > int32(manager.InterceptDispositionType_WAITING) { @@ -132,7 +140,7 @@ func (ii *Info) WriteTo(w io.Writer) (int64, error) { pkv := ioutil.DefaultKeyValueFormatter() pkv.Indent = "" pkv.Separator = " -> " - if ii.PortID != "" { + if ii.ContainerPort != 0 { pm, _ := agentconfig.NewPortIdentifier(ii.Protocol, strconv.Itoa(int(ii.ContainerPort))) pkv.Add(pm.String(), fmt.Sprintf("%d %s", ii.TargetPort, ii.Protocol)) } @@ -141,7 +149,11 @@ func (ii *Info) WriteTo(w io.Writer) (int64, error) { to := pm.To() pkv.Add(pm.From().String(), fmt.Sprintf("%d %s", to.Port, to.Proto)) } - kvf.Add("Intercepting", fmt.Sprintf("%s -> %s\n%s", ii.PodIP, ii.TargetHost, pkv)) + key := "Intercepting" + if ii.Replace { + key = "Port forwards" + } + kvf.Add(key, fmt.Sprintf("%s -> %s\n%s", ii.PodIP, ii.TargetHost, pkv)) if !ii.Global { kvf.Add("Intercepting", func() string { diff --git a/pkg/client/cli/intercept/state.go b/pkg/client/cli/intercept/state.go index 013792e5b8..1abeb35d12 100644 --- a/pkg/client/cli/intercept/state.go +++ b/pkg/client/cli/intercept/state.go @@ -84,6 +84,7 @@ func (s *state) CreateRequest(ctx context.Context) (*connector.CreateInterceptRe spec.MechanismArgs = s.MechanismArgs spec.Agent = s.AgentName spec.TargetHost = "127.0.0.1" + spec.NoDefaultPort = s.NoDefaultPort ud := daemon.GetUserClient(ctx) diff --git a/pkg/client/userd/trafficmgr/intercept.go b/pkg/client/userd/trafficmgr/intercept.go index a9e5a85911..f3a28e1520 100644 --- a/pkg/client/userd/trafficmgr/intercept.go +++ b/pkg/client/userd/trafficmgr/intercept.go @@ -375,7 +375,7 @@ func ensureUniqueLocalPorts(spec *manager.InterceptSpec, pi *manager.PreparedInt targetPort = pi.ContainerPort } - ports := make(map[agentconfig.PortAndProto]struct{}, len(spec.LocalPorts)+len(spec.PodPorts)+1) + ports := make(map[agentconfig.PortAndProto]struct{}, len(spec.LocalPorts)+len(pi.PodPorts)+1) ports[agentconfig.PortAndProto{ Port: uint16(targetPort), Proto: core.Protocol(pi.Protocol), @@ -518,7 +518,9 @@ func (s *session) AddIntercept(c context.Context, ir *rpc.CreateInterceptRequest // of using PrepareIntercept. pi := iInfo.PreparedIntercept() - if pi.ServicePort > 0 || pi.ServicePortName != "" { + if spec.PortIdentifier == "all" { + spec.PortIdentifier = "" + } else if pi.ServicePort > 0 || pi.ServicePortName != "" { // Make spec port identifier unambiguous. spec.ServicePortName = pi.ServicePortName spec.ServicePort = pi.ServicePort @@ -531,6 +533,10 @@ func (s *session) AddIntercept(c context.Context, ir *rpc.CreateInterceptRequest dlog.Debugf(c, "pi.Protocol = %s", pi.Protocol) spec.Protocol = pi.Protocol spec.ContainerPort = pi.ContainerPort + spec.ContainerName = pi.ContainerName + if spec.NoDefaultPort { + spec.Name = spec.Agent + "/" + pi.ContainerName + } spec.PodPorts = pi.PodPorts result = iInfo.InterceptResult() @@ -754,15 +760,48 @@ func (s *session) GetInterceptInfo(name string) *manager.InterceptInfo { } // GetInterceptSpec returns the InterceptSpec for the given name, or nil if no such spec exists. -func (s *session) getInterceptByName(name string) (found *intercept) { +func (s *session) getInterceptByName(name string) *intercept { s.currentInterceptsLock.Lock() + defer s.currentInterceptsLock.Unlock() for _, ic := range s.currentIntercepts { if ic.Spec.Name == name { + return ic + } + } + + if slashIx := strings.IndexByte(name, '/'); slashIx > 0 { + container := name[slashIx+1:] + name = name[:slashIx] + for _, ic := range s.currentIntercepts { + if ic.Spec.Name == name && container == ic.Spec.ContainerName { + return ic + } + } + return nil + } + + // Check if the name uniquely identifies a `replace` by its workload (always uses /) + namePfx := name + "/" + var found *intercept + for _, ic := range s.currentIntercepts { + if strings.HasPrefix(ic.Spec.Name, namePfx) { + if found != nil { + // Found a second time using prefix, so the prefix isn't unique and hence not valid. + return nil + } found = ic - break } } - s.currentInterceptsLock.Unlock() + if found != nil { + // Name is not unique if it also identifies an ingest with the same workload. + s.currentIngests.Range(func(key ingestKey, ig *ingest) bool { + if key.workload == name { + found = nil + return false + } + return true + }) + } return found } diff --git a/pkg/client/userd/trafficmgr/session.go b/pkg/client/userd/trafficmgr/session.go index cc3649df35..a3559c4a99 100644 --- a/pkg/client/userd/trafficmgr/session.go +++ b/pkg/client/userd/trafficmgr/session.go @@ -590,6 +590,7 @@ func (s *session) ApplyConfig(ctx context.Context) error { // getInfosForWorkloads returns a list of workloads found in the given namespace that fulfils the given filter criteria. func (s *session) getInfosForWorkloads( + ctx context.Context, namespaces []string, iMap map[string][]*manager.InterceptInfo, gMap map[string][]*rpc.IngestInfo, @@ -611,8 +612,16 @@ func (s *session) getInfosForWorkloads( var ok bool filterMatch := rpc.ListRequest_EVERYTHING - if wlInfo.InterceptInfos, ok = iMap[name]; !ok { - filterMatch &= ^rpc.ListRequest_INTERCEPTS + + filterMatch &= ^(rpc.ListRequest_REPLACEMENTS | rpc.ListRequest_INTERCEPTS) + if wlInfo.InterceptInfos, ok = iMap[name]; ok { + for _, ii := range wlInfo.InterceptInfos { + if ii.Spec.NoDefaultPort { + filterMatch |= rpc.ListRequest_REPLACEMENTS + } else { + filterMatch |= rpc.ListRequest_INTERCEPTS + } + } } if wlInfo.IngestInfos, ok = gMap[name]; !ok { filterMatch &= ^rpc.ListRequest_INGESTS @@ -620,6 +629,7 @@ func (s *session) getInfosForWorkloads( if wlInfo.AgentVersion, ok = sMap[name]; !ok { filterMatch &= ^rpc.ListRequest_INSTALLED_AGENTS } + dlog.Debugf(ctx, "filter %d, filterMatch %d", filter, filterMatch) if filter != 0 && filter&filterMatch == 0 { return } @@ -652,7 +662,7 @@ func (s *session) WatchWorkloads(c context.Context, wr *rpc.WatchWorkloadsReques }() send := func() error { - ws, err := s.WorkloadInfoSnapshot(c, wr.Namespaces, rpc.ListRequest_EVERYTHING) + ws, err := s.WorkloadInfoSnapshot(c, wr.Namespaces, rpc.ListRequest_UNSPECIFIED) if err != nil { return err } @@ -718,16 +728,11 @@ func (s *session) WorkloadInfoSnapshot( var nss []string var sMap map[string]string - if filter&(rpc.ListRequest_INTERCEPTS|rpc.ListRequest_INGESTS|rpc.ListRequest_INSTALLED_AGENTS) != 0 { - // Special case, we don't care about namespaces in general. Instead, we use the connected namespace - nss = []string{s.Namespace} - } else { - nss = make([]string, 0, len(namespaces)) - for _, ns := range namespaces { - ns = s.ActualNamespace(ns) - if ns != "" { - nss = append(nss, ns) - } + nss = make([]string, 0, len(namespaces)) + for _, ns := range namespaces { + ns = s.ActualNamespace(ns) + if ns != "" { + nss = append(nss, ns) } } if len(nss) == 0 { @@ -759,7 +764,7 @@ nextIs: return true }) - workloadInfos := s.getInfosForWorkloads(nss, iMap, gMap, sMap, filter) + workloadInfos := s.getInfosForWorkloads(ctx, nss, iMap, gMap, sMap, filter) return &rpc.WorkloadInfoSnapshot{Workloads: workloadInfos}, nil } diff --git a/rpc/connector/connector.pb.go b/rpc/connector/connector.pb.go index 4d371dd318..651e160a9a 100644 --- a/rpc/connector/connector.pb.go +++ b/rpc/connector/connector.pb.go @@ -154,9 +154,9 @@ type ListRequest_Filter int32 const ( ListRequest_UNSPECIFIED ListRequest_Filter = 0 ListRequest_INTERCEPTS ListRequest_Filter = 1 - ListRequest_INGESTS ListRequest_Filter = 2 - ListRequest_INSTALLED_AGENTS ListRequest_Filter = 4 - ListRequest_INTERCEPTABLE ListRequest_Filter = 8 + ListRequest_REPLACEMENTS ListRequest_Filter = 2 + ListRequest_INGESTS ListRequest_Filter = 4 + ListRequest_INSTALLED_AGENTS ListRequest_Filter = 8 ListRequest_EVERYTHING ListRequest_Filter = 15 ) @@ -165,17 +165,17 @@ var ( ListRequest_Filter_name = map[int32]string{ 0: "UNSPECIFIED", 1: "INTERCEPTS", - 2: "INGESTS", - 4: "INSTALLED_AGENTS", - 8: "INTERCEPTABLE", + 2: "REPLACEMENTS", + 4: "INGESTS", + 8: "INSTALLED_AGENTS", 15: "EVERYTHING", } ListRequest_Filter_value = map[string]int32{ "UNSPECIFIED": 0, "INTERCEPTS": 1, - "INGESTS": 2, - "INSTALLED_AGENTS": 4, - "INTERCEPTABLE": 8, + "REPLACEMENTS": 2, + "INGESTS": 4, + "INSTALLED_AGENTS": 8, "EVERYTHING": 15, } ) @@ -1971,380 +1971,380 @@ var file_connector_connector_proto_rawDesc = string([]byte{ 0x6c, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x61, 0x64, 0x4f, 0x6e, - 0x6c, 0x79, 0x22, 0xe0, 0x01, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x6c, 0x79, 0x22, 0xdf, 0x01, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x22, 0x6f, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x0f, + 0x70, 0x61, 0x63, 0x65, 0x22, 0x6e, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x43, 0x45, 0x50, 0x54, 0x53, 0x10, 0x01, 0x12, - 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x47, 0x45, 0x53, 0x54, 0x53, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, - 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x45, 0x44, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x53, - 0x10, 0x04, 0x12, 0x11, 0x0a, 0x0d, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x43, 0x45, 0x50, 0x54, 0x41, - 0x42, 0x4c, 0x45, 0x10, 0x08, 0x12, 0x0e, 0x0a, 0x0a, 0x45, 0x56, 0x45, 0x52, 0x59, 0x54, 0x48, - 0x49, 0x4e, 0x47, 0x10, 0x0f, 0x22, 0x5e, 0x0a, 0x10, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, - 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x25, - 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xc5, 0x01, 0x0a, 0x0d, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, - 0x69, 0x66, 0x69, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x64, 0x65, 0x6e, 0x74, - 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, - 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, - 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x6d, 0x6f, 0x75, 0x6e, - 0x74, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6c, 0x6f, - 0x63, 0x61, 0x6c, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0a, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, 0xa0, 0x03, - 0x0a, 0x0a, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1a, 0x0a, 0x08, - 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, - 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x1c, 0x0a, - 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, - 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x15, 0x0a, 0x06, - 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, - 0x64, 0x49, 0x70, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, - 0x12, 0x19, 0x0a, 0x08, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x07, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x55, 0x0a, 0x0b, 0x65, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x33, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, - 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x6f, 0x75, - 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, - 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, - 0x1a, 0x3e, 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x37, 0x0a, 0x15, 0x57, 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, - 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x61, 0x6d, - 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6e, - 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x22, 0xfc, 0x02, 0x0a, 0x0c, 0x57, 0x6f, - 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, - 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x38, 0x0a, 0x18, - 0x6e, 0x6f, 0x74, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, - 0x6e, 0x6f, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x4c, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, - 0x65, 0x70, 0x74, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x10, 0x0a, 0x0c, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x4d, 0x45, 0x4e, 0x54, 0x53, 0x10, + 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x47, 0x45, 0x53, 0x54, 0x53, 0x10, 0x04, 0x12, 0x14, + 0x0a, 0x10, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x45, 0x44, 0x5f, 0x41, 0x47, 0x45, 0x4e, + 0x54, 0x53, 0x10, 0x08, 0x12, 0x0e, 0x0a, 0x0a, 0x45, 0x56, 0x45, 0x52, 0x59, 0x54, 0x48, 0x49, + 0x4e, 0x47, 0x10, 0x0f, 0x22, 0x5e, 0x0a, 0x10, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, + 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, + 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xc5, 0x01, 0x0a, 0x0d, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x66, 0x69, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x66, 0x69, 0x65, 0x72, 0x52, 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, + 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, + 0x74, 0x12, 0x28, 0x0a, 0x10, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x6d, 0x6f, 0x75, 0x6e, 0x74, + 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6c, 0x6f, 0x63, + 0x61, 0x6c, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0a, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, 0xa0, 0x03, 0x0a, + 0x0a, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1a, 0x0a, 0x08, 0x77, + 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, + 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, + 0x6f, 0x61, 0x64, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x1c, 0x0a, 0x09, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x6f, + 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x15, 0x0a, 0x06, 0x70, + 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x64, + 0x49, 0x70, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, + 0x19, 0x0a, 0x08, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x07, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x55, 0x0a, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x33, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x6f, 0x75, 0x6e, + 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x1a, + 0x3e, 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x37, 0x0a, 0x15, 0x57, 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x61, 0x6d, 0x65, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x22, 0xfc, 0x02, 0x0a, 0x0c, 0x57, 0x6f, 0x72, + 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, + 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x38, 0x0a, 0x18, 0x6e, + 0x6f, 0x74, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x6e, + 0x6f, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, + 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x4c, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, + 0x70, 0x74, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, + 0x66, 0x6f, 0x73, 0x12, 0x45, 0x0a, 0x0c, 0x69, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x6e, + 0x66, 0x6f, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0b, 0x69, + 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x77, 0x6f, + 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x77, 0x6f, 0x72, 0x6b, + 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, + 0x69, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x5a, 0x0a, 0x14, 0x57, 0x6f, 0x72, 0x6b, 0x6c, + 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, + 0x42, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, + 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, + 0x61, 0x64, 0x73, 0x22, 0xaa, 0x02, 0x0a, 0x0f, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, + 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x4a, 0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x63, 0x65, 0x70, 0x74, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x45, 0x0a, 0x0c, 0x69, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x5f, 0x69, - 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0b, - 0x69, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x77, - 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x77, 0x6f, 0x72, - 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x75, 0x69, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, - 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x5a, 0x0a, 0x14, 0x57, 0x6f, 0x72, 0x6b, - 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x12, 0x42, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x57, 0x6f, 0x72, - 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x6c, - 0x6f, 0x61, 0x64, 0x73, 0x22, 0xaa, 0x02, 0x0a, 0x0f, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, - 0x70, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x4a, 0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, - 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x39, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, - 0x65, 0x70, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, - 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x54, 0x65, 0x78, 0x74, 0x12, 0x25, - 0x0a, 0x0e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x61, 0x74, - 0x65, 0x67, 0x6f, 0x72, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x5f, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x55, 0x69, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, - 0x61, 0x64, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, - 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x4a, 0x04, 0x08, 0x04, 0x10, - 0x05, 0x22, 0xe5, 0x01, 0x0a, 0x0f, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, - 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, - 0x65, 0x6c, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x43, 0x0a, 0x05, 0x73, 0x63, 0x6f, - 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x22, 0x39, - 0x0a, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, - 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x4c, 0x4f, 0x43, 0x41, - 0x4c, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x45, 0x4d, 0x4f, - 0x54, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x22, 0x8f, 0x01, 0x0a, 0x0b, 0x4c, 0x6f, - 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x74, 0x72, 0x61, - 0x66, 0x66, 0x69, 0x63, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0e, 0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x4d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x12, 0x20, 0x0a, 0x0c, 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x79, 0x61, - 0x6d, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x67, 0x65, 0x74, 0x50, 0x6f, 0x64, - 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1d, 0x0a, 0x0a, - 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x44, 0x69, 0x72, 0x22, 0xae, 0x01, 0x0a, 0x0c, - 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x12, 0x4c, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4c, 0x6f, - 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x6f, 0x64, 0x49, 0x6e, - 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, - 0x1a, 0x3a, 0x0a, 0x0c, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5a, 0x0a, 0x14, - 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x66, 0x6f, 0x72, 0x5f, 0x63, 0x6c, 0x69, 0x65, - 0x6e, 0x74, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0f, 0x66, 0x6f, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0x37, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x4e, - 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x73, 0x22, 0x22, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x04, 0x6a, 0x73, 0x6f, 0x6e, 0x22, 0x8c, 0x01, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x3c, 0x0a, 0x0b, 0x70, 0x6f, 0x64, 0x5f, - 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x0a, 0x70, 0x6f, 0x64, 0x53, - 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x3c, 0x0a, 0x0b, 0x73, 0x76, 0x63, 0x5f, 0x73, 0x75, - 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x0a, 0x73, 0x76, 0x63, 0x53, 0x75, 0x62, - 0x6e, 0x65, 0x74, 0x73, 0x32, 0x9e, 0x15, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x12, 0x43, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4d, 0x0a, 0x11, 0x52, 0x6f, 0x6f, 0x74, 0x44, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x67, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x39, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, + 0x70, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1d, + 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x54, 0x65, 0x78, 0x74, 0x12, 0x25, 0x0a, + 0x0e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x61, 0x74, 0x65, + 0x67, 0x6f, 0x72, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, + 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x55, 0x69, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, + 0x64, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x6f, + 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, + 0x22, 0xe5, 0x01, 0x0a, 0x0f, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, + 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x43, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x22, 0x39, 0x0a, + 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x4c, 0x4f, 0x43, 0x41, 0x4c, + 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x45, 0x4d, 0x4f, 0x54, + 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x22, 0x8f, 0x01, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x74, 0x72, 0x61, 0x66, + 0x66, 0x69, 0x63, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0e, 0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x12, 0x20, 0x0a, 0x0c, 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x79, 0x61, 0x6d, + 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x67, 0x65, 0x74, 0x50, 0x6f, 0x64, 0x59, + 0x61, 0x6d, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x65, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x44, 0x69, 0x72, 0x22, 0xae, 0x01, 0x0a, 0x0c, 0x4c, + 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x12, 0x4c, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4c, 0x6f, 0x67, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, + 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x1a, + 0x3a, 0x0a, 0x0c, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5a, 0x0a, 0x14, 0x47, + 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x66, 0x6f, 0x72, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, + 0x66, 0x6f, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, + 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0x37, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x4e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, + 0x22, 0x22, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x12, 0x0a, 0x04, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x6a, 0x73, 0x6f, 0x6e, 0x22, 0x8c, 0x01, 0x0a, 0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x3c, 0x0a, 0x0b, 0x70, 0x6f, 0x64, 0x5f, 0x73, + 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x0a, 0x70, 0x6f, 0x64, 0x53, 0x75, + 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x3c, 0x0a, 0x0b, 0x73, 0x76, 0x63, 0x5f, 0x73, 0x75, 0x62, + 0x6e, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x0a, 0x73, 0x76, 0x63, 0x53, 0x75, 0x62, 0x6e, + 0x65, 0x74, 0x73, 0x32, 0x9e, 0x15, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x12, 0x43, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x51, 0x0a, 0x15, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, - 0x63, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4c, 0x0a, 0x0d, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, - 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x5e, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x47, - 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, - 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x56, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x12, 0x26, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x3c, 0x0a, 0x0a, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, - 0x11, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x75, 0x62, 0x6e, 0x65, - 0x74, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x26, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x75, 0x62, 0x6e, 0x65, - 0x74, 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x67, 0x0a, 0x0c, 0x43, 0x61, 0x6e, - 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4d, 0x0a, 0x11, 0x52, 0x6f, 0x6f, 0x74, 0x44, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x1a, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x51, 0x0a, 0x15, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, + 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4c, 0x0a, 0x0d, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, + 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x5e, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x47, 0x65, + 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, + 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x56, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x12, 0x26, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, - 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x6f, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3c, + 0x0a, 0x0a, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, 0x11, + 0x47, 0x65, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, + 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x26, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x12, 0x53, 0x0a, 0x06, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x12, 0x25, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, - 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x59, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x49, 0x6e, - 0x67, 0x65, 0x73, 0x74, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, - 0x67, 0x65, 0x73, 0x74, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x1a, 0x22, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x6e, - 0x66, 0x6f, 0x12, 0x5b, 0x0a, 0x0b, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x49, 0x6e, 0x67, 0x65, 0x73, - 0x74, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, - 0x74, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x1a, 0x22, 0x2e, 0x74, 0x65, + 0x6f, 0x72, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, + 0x73, 0x12, 0x45, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x67, 0x0a, 0x0c, 0x43, 0x61, 0x6e, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x12, 0x53, 0x0a, 0x06, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x12, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x6a, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, - 0x70, 0x74, 0x12, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x69, 0x0a, 0x0f, 0x52, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2d, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0x1a, 0x27, 0x2e, + 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, + 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x59, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x67, + 0x65, 0x73, 0x74, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, + 0x65, 0x73, 0x74, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x64, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x52, 0x0a, 0x09, - 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x12, 0x59, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x5b, 0x0a, 0x0b, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, + 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, + 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x6a, + 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, + 0x74, 0x12, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x69, 0x0a, 0x0f, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2d, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0x1a, 0x27, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x64, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x52, 0x0a, 0x09, 0x55, + 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, - 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x6f, 0x0a, 0x0e, 0x57, - 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x12, 0x2d, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, - 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x74, + 0x72, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, + 0x59, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, - 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x4e, 0x0a, 0x0b, - 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x27, 0x2e, 0x74, 0x65, + 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x6f, 0x0a, 0x0e, 0x57, 0x61, + 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x12, 0x2d, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, + 0x6f, 0x61, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x36, 0x0a, 0x04, - 0x51, 0x75, 0x69, 0x74, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0a, 0x47, 0x61, 0x74, 0x68, 0x65, 0x72, 0x4c, 0x6f, - 0x67, 0x73, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, - 0x0e, 0x41, 0x64, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x6f, 0x72, 0x12, - 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, - 0x70, 0x74, 0x6f, 0x72, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x50, 0x0a, 0x11, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x6f, - 0x72, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x63, 0x65, 0x70, 0x74, 0x6f, 0x72, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x6c, - 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, - 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x59, 0x0a, 0x15, - 0x47, 0x65, 0x74, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, - 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x28, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, - 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x4e, 0x0a, 0x17, 0x52, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, - 0x74, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, - 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x49, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x24, 0x2e, 0x74, + 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, + 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x4e, 0x0a, 0x0b, 0x53, + 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x27, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x36, 0x0a, 0x04, 0x51, + 0x75, 0x69, 0x74, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0a, 0x47, 0x61, 0x74, 0x68, 0x65, 0x72, 0x4c, 0x6f, 0x67, + 0x73, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, + 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x0e, + 0x41, 0x64, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x23, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, + 0x74, 0x6f, 0x72, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x50, 0x0a, 0x11, 0x52, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x6f, 0x72, + 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, + 0x65, 0x70, 0x74, 0x6f, 0x72, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x6c, 0x0a, + 0x0d, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x2c, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x54, 0x0a, 0x0e, 0x53, 0x65, 0x74, 0x44, 0x4e, 0x53, 0x45, 0x78, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x73, 0x12, 0x2a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x44, 0x4e, - 0x53, 0x45, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x54, 0x0a, 0x0e, 0x53, 0x65, 0x74, 0x44, - 0x4e, 0x53, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x2a, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x53, 0x65, 0x74, 0x44, 0x4e, 0x53, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x65, - 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x89, 0x04, 0x0a, 0x0c, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x59, 0x0a, 0x15, 0x47, + 0x65, 0x74, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, + 0x69, 0x6e, 0x64, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x28, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, + 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x4e, 0x0a, 0x17, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, + 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, + 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x49, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x24, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x54, 0x0a, 0x0e, 0x53, 0x65, 0x74, 0x44, 0x4e, 0x53, 0x45, 0x78, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x73, 0x12, 0x2a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x44, 0x4e, 0x53, + 0x45, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x54, 0x0a, 0x0e, 0x53, 0x65, 0x74, 0x44, 0x4e, + 0x53, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x2a, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x53, 0x65, 0x74, 0x44, 0x4e, 0x53, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x65, 0x0a, + 0x0e, 0x47, 0x65, 0x74, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x4a, 0x0a, - 0x0f, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x89, 0x04, 0x0a, 0x0c, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x43, 0x4c, 0x49, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x60, 0x0a, 0x0b, 0x45, 0x6e, 0x73, - 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x5a, 0x0a, 0x10, 0x57, - 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, - 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, 0x12, 0x50, 0x0a, 0x09, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, - 0x70, 0x44, 0x4e, 0x53, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, - 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x56, 0x0a, 0x06, 0x54, 0x75, 0x6e, - 0x6e, 0x65, 0x6c, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, - 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, - 0x01, 0x42, 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x69, 0x6f, 0x2f, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, - 0x76, 0x32, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x4a, 0x0a, 0x0f, + 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, + 0x4c, 0x49, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x60, 0x0a, 0x0b, 0x45, 0x6e, 0x73, 0x75, + 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x45, + 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, + 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x5a, 0x0a, 0x10, 0x57, 0x61, + 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x21, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, 0x12, 0x50, 0x0a, 0x09, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, + 0x44, 0x4e, 0x53, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x56, 0x0a, 0x06, 0x54, 0x75, 0x6e, 0x6e, + 0x65, 0x6c, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, + 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, 0x01, + 0x42, 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x69, 0x6f, 0x2f, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x76, + 0x32, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, }) var ( diff --git a/rpc/connector/connector.proto b/rpc/connector/connector.proto index f965e1b44e..8ce7e00ede 100644 --- a/rpc/connector/connector.proto +++ b/rpc/connector/connector.proto @@ -279,9 +279,9 @@ message ListRequest { enum Filter { UNSPECIFIED = 0; INTERCEPTS = 1; - INGESTS = 2; - INSTALLED_AGENTS = 4; - INTERCEPTABLE = 8; + REPLACEMENTS = 2; + INGESTS = 4; + INSTALLED_AGENTS = 8; EVERYTHING = 15; } Filter filter = 1; diff --git a/rpc/manager/manager.pb.go b/rpc/manager/manager.pb.go index f7cb4a1514..cefa2ab0e2 100644 --- a/rpc/manager/manager.pb.go +++ b/rpc/manager/manager.pb.go @@ -683,7 +683,9 @@ type InterceptSpec struct { // Deprecated: use local_ports instead ExtraPorts []int32 `protobuf:"varint,15,rep,packed,name=extra_ports,json=extraPorts,proto3" json:"extra_ports,omitempty"` // Whether to replace the running container. - Replace bool `protobuf:"varint,22,opt,name=replace,proto3" json:"replace,omitempty"` + Replace bool `protobuf:"varint,22,opt,name=replace,proto3" json:"replace,omitempty"` + // Intercept desire no default port. + NoDefaultPort bool `protobuf:"varint,25,opt,name=no_default_port,json=noDefaultPort,proto3" json:"no_default_port,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -879,6 +881,13 @@ func (x *InterceptSpec) GetReplace() bool { return false } +func (x *InterceptSpec) GetNoDefaultPort() bool { + if x != nil { + return x.NoDefaultPort + } + return false +} + type IngressInfo struct { state protoimpl.MessageState `protogen:"open.v1"` // The layer-3 host @@ -4096,7 +4105,7 @@ var file_manager_manager_proto_rawDesc = string([]byte{ 0x31, 0x0a, 0x0b, 0x50, 0x6f, 0x72, 0x74, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x74, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, - 0x74, 0x6f, 0x22, 0x90, 0x06, 0x0a, 0x0d, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x74, 0x6f, 0x22, 0xb8, 0x06, 0x0a, 0x0d, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, @@ -4144,754 +4153,756 @@ var file_manager_manager_proto_rawDesc = string([]byte{ 0x65, 0x6f, 0x75, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0a, 0x65, 0x78, 0x74, 0x72, 0x61, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, - 0x18, 0x16, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4a, - 0x04, 0x08, 0x0b, 0x10, 0x0c, 0x22, 0x66, 0x0a, 0x0b, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x17, 0x0a, 0x07, - 0x75, 0x73, 0x65, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x75, - 0x73, 0x65, 0x54, 0x6c, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x35, 0x68, 0x6f, 0x73, 0x74, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x35, 0x68, 0x6f, 0x73, 0x74, 0x22, 0xcb, 0x02, - 0x0a, 0x0b, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3b, 0x0a, - 0x07, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, + 0x18, 0x16, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x12, + 0x26, 0x0a, 0x0f, 0x6e, 0x6f, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x70, 0x6f, + 0x72, 0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x6f, 0x44, 0x65, 0x66, 0x61, + 0x75, 0x6c, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x4a, 0x04, 0x08, 0x0b, 0x10, 0x0c, 0x22, 0x66, 0x0a, + 0x0b, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, + 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, + 0x70, 0x6f, 0x72, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x5f, 0x74, 0x6c, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x75, 0x73, 0x65, 0x54, 0x6c, 0x73, 0x12, 0x16, 0x0a, + 0x06, 0x6c, 0x35, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, + 0x35, 0x68, 0x6f, 0x73, 0x74, 0x22, 0xcb, 0x02, 0x0a, 0x0b, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, + 0x77, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3b, 0x0a, 0x07, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, + 0x67, 0x72, 0x65, 0x73, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x69, 0x6e, 0x67, 0x72, 0x65, + 0x73, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x62, 0x61, + 0x6e, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x64, 0x69, 0x73, 0x70, + 0x6c, 0x61, 0x79, 0x42, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x75, 0x6c, + 0x6c, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x75, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x55, 0x72, 0x6c, 0x12, 0x68, 0x0a, 0x13, 0x61, 0x64, 0x64, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x38, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, + 0x70, 0x65, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x61, 0x64, 0x64, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x1a, 0x44, 0x0a, + 0x16, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0x81, 0x09, 0x0a, 0x0d, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, + 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x63, 0x65, 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x48, + 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, + 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x76, 0x69, + 0x65, 0x77, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x44, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x76, + 0x69, 0x65, 0x77, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x07, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x69, - 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x62, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0d, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x42, 0x61, 0x6e, 0x6e, 0x65, - 0x72, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x75, 0x6c, 0x6c, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x75, 0x6c, - 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x55, 0x72, 0x6c, 0x12, 0x68, 0x0a, 0x13, 0x61, - 0x64, 0x64, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x11, 0x61, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x73, 0x1a, 0x44, 0x0a, 0x16, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x81, 0x09, 0x0a, 0x0d, - 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, - 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, - 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x48, 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, + 0x63, 0x52, 0x0b, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x12, 0x50, + 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x63, 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x6f, + 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x6f, + 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x69, 0x5f, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x61, 0x70, 0x69, 0x50, 0x6f, 0x72, 0x74, + 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x70, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x66, 0x74, 0x70, 0x5f, + 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x66, 0x74, 0x70, + 0x50, 0x6f, 0x72, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, + 0x18, 0x12, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, + 0x2c, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1f, 0x0a, + 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x10, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2e, + 0x0a, 0x13, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x5f, 0x61, 0x72, 0x67, 0x73, + 0x5f, 0x64, 0x65, 0x73, 0x63, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6d, 0x65, 0x63, + 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x41, 0x72, 0x67, 0x73, 0x44, 0x65, 0x73, 0x63, 0x12, 0x4a, + 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x30, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x4d, 0x0a, 0x08, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, + 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x56, 0x0a, 0x0b, 0x65, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x11, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x0d, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, 0x65, - 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x12, 0x44, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x73, 0x70, 0x65, 0x63, - 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, - 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0b, 0x70, 0x72, 0x65, 0x76, 0x69, - 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x12, 0x50, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, - 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x64, 0x69, 0x73, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x13, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x19, 0x0a, - 0x08, 0x61, 0x70, 0x69, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x07, 0x61, 0x70, 0x69, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x5f, - 0x69, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x70, 0x12, - 0x1b, 0x0a, 0x09, 0x73, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0b, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x08, 0x73, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x19, 0x0a, 0x08, - 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x12, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, - 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x5f, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x75, 0x6e, 0x74, - 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, - 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x13, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, - 0x69, 0x73, 0x6d, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x18, 0x0c, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x11, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x41, 0x72, - 0x67, 0x73, 0x44, 0x65, 0x73, 0x63, 0x12, 0x4a, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x48, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x73, 0x12, 0x4d, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x0f, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x12, 0x56, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x18, 0x11, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x6e, 0x76, 0x69, - 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, 0x6e, - 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x0b, 0x6d, 0x6f, 0x64, - 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x69, - 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, - 0x3e, 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x8d, 0x01, 0x0a, 0x0b, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x2c, - 0x0a, 0x12, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, - 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0a, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x48, 0x00, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x49, 0x64, 0x88, 0x01, 0x01, - 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x22, - 0x6c, 0x0a, 0x0d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, - 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x22, 0x4c, 0x0a, - 0x11, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, - 0x6f, 0x74, 0x12, 0x37, 0x0a, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x5c, 0x0a, 0x15, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, - 0x73, 0x68, 0x6f, 0x74, 0x12, 0x43, 0x0a, 0x0a, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, - 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x73, 0x22, 0xba, 0x01, 0x0a, 0x16, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x4a, 0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x5f, 0x73, - 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x12, 0x3b, 0x0a, 0x0b, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, + 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x1a, 0x3a, + 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8d, 0x01, 0x0a, 0x0b, 0x53, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x10, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x49, 0x6e, 0x73, 0x74, 0x61, + 0x6c, 0x6c, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, + 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, + 0x61, 0x6c, 0x6c, 0x49, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x69, 0x6e, 0x73, + 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x22, 0x6c, 0x0a, 0x0d, 0x41, 0x67, 0x65, 0x6e, 0x74, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0d, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x17, 0x0a, - 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x22, 0x65, 0x0a, 0x12, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, + 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x73, 0x22, 0x4c, 0x0a, 0x11, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, + 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x37, 0x0a, 0x06, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x61, 0x67, 0x65, + 0x6e, 0x74, 0x73, 0x22, 0x5c, 0x0a, 0x15, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x43, 0x0a, 0x0a, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, + 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x73, 0x22, 0xba, 0x01, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xce, 0x03, - 0x0a, 0x11, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, - 0x65, 0x70, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x5f, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, - 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1f, - 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x69, 0x64, 0x12, - 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x6f, - 0x72, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, - 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, - 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6b, 0x69, - 0x6e, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, - 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, - 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x67, 0x65, - 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0c, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, - 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x0d, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, 0x8b, - 0x02, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, - 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x51, 0x0a, 0x12, 0x61, 0x64, - 0x64, 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, - 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x48, 0x00, 0x52, 0x10, 0x61, 0x64, 0x64, - 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x34, 0x0a, - 0x15, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, - 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x13, - 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x44, 0x6f, 0x6d, - 0x61, 0x69, 0x6e, 0x42, 0x17, 0x0a, 0x15, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x6a, 0x0a, 0x17, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x66, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4a, 0x0a, 0x0e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, + 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, + 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x22, 0x65, + 0x0a, 0x12, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xce, 0x03, 0x0a, 0x11, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, + 0x65, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x61, 0x74, 0x65, 0x67, + 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x43, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, + 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, + 0x72, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, + 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x1f, + 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, + 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x25, 0x0a, 0x0e, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, + 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x64, + 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, + 0x64, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, 0x8b, 0x02, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x51, 0x0a, 0x12, 0x61, 0x64, 0x64, 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, + 0x77, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, + 0x63, 0x48, 0x00, 0x52, 0x10, 0x61, 0x64, 0x64, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x44, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, + 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x13, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x72, + 0x65, 0x76, 0x69, 0x65, 0x77, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x42, 0x17, 0x0a, 0x15, 0x70, + 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x61, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x6a, 0x0a, 0x17, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x22, 0xb8, 0x06, 0x0a, 0x16, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x50, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, - 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x44, 0x69, - 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x64, - 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x70, 0x12, 0x1b, 0x0a, 0x09, 0x73, - 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, - 0x73, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x74, 0x70, 0x5f, - 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x66, 0x74, 0x70, 0x50, - 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, - 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x13, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, - 0x6d, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x11, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x41, 0x72, 0x67, 0x73, - 0x44, 0x65, 0x73, 0x63, 0x12, 0x53, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, - 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, + 0x22, 0x66, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xb8, 0x06, 0x0a, 0x16, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x56, 0x0a, 0x08, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, - 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x12, 0x5f, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, - 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3b, - 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x45, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x65, 0x0a, 0x0d, 0x52, - 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, - 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, - 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, - 0x65, 0x79, 0x22, 0x65, 0x0a, 0x0f, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, - 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, - 0x65, 0x6c, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x73, 0x0a, 0x0e, 0x47, 0x65, 0x74, - 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x74, - 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x4d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x20, 0x0a, 0x0c, - 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0a, 0x67, 0x65, 0x74, 0x50, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0xb7, - 0x02, 0x0a, 0x0c, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4a, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x6f, 0x64, 0x4c, 0x6f, 0x67, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, - 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, - 0x72, 0x4d, 0x73, 0x67, 0x12, 0x4a, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x79, 0x61, 0x6d, 0x6c, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, - 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x6f, 0x64, 0x59, 0x61, - 0x6d, 0x6c, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, - 0x1a, 0x3a, 0x0a, 0x0c, 0x50, 0x6f, 0x64, 0x4c, 0x6f, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3a, 0x0a, 0x0c, - 0x50, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x29, 0x0a, 0x13, 0x54, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x41, 0x50, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, - 0x6f, 0x72, 0x74, 0x22, 0x3c, 0x0a, 0x0c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, - 0x66, 0x6f, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x22, 0x85, 0x01, 0x0a, 0x07, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, - 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, - 0x4d, 0x73, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x22, 0x6c, 0x0a, 0x15, 0x41, 0x6d, 0x62, - 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x1e, 0x0a, 0x08, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x5f, 0x63, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x07, - 0x70, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x61, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, - 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x63, 0x61, 0x22, 0x3c, 0x0a, 0x19, 0x41, 0x6d, 0x62, 0x61, 0x73, - 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x61, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x63, 0x61, 0x6e, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x22, 0x29, 0x0a, 0x0d, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, - 0x22, 0x7c, 0x0a, 0x0b, 0x44, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x17, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x6e, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x6f, 0x75, 0x6e, - 0x64, 0x74, 0x72, 0x69, 0x70, 0x5f, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x10, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x74, 0x72, 0x69, 0x70, 0x4c, 0x61, - 0x74, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x69, 0x61, 0x6c, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x64, 0x69, 0x61, - 0x6c, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x71, - 0x0a, 0x0a, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, - 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, + 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x50, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x15, 0x0a, 0x06, + 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, + 0x64, 0x49, 0x70, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, + 0x12, 0x19, 0x0a, 0x08, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0c, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x07, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, + 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x13, + 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x64, + 0x65, 0x73, 0x63, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6d, 0x65, 0x63, 0x68, 0x61, + 0x6e, 0x69, 0x73, 0x6d, 0x41, 0x72, 0x67, 0x73, 0x44, 0x65, 0x73, 0x63, 0x12, 0x53, 0x0a, 0x07, + 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x22, 0x36, 0x0a, 0x0b, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x15, 0x0a, 0x06, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x05, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x72, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x72, 0x72, 0x73, 0x22, 0xca, 0x01, 0x0a, 0x10, 0x44, 0x4e, - 0x53, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, - 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, - 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x07, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, - 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x08, 0x72, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2b, 0x0a, 0x05, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x70, 0x12, - 0x12, 0x0a, 0x04, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6d, - 0x61, 0x73, 0x6b, 0x22, 0x8c, 0x04, 0x0a, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x42, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x73, - 0x75, 0x62, 0x6e, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x12, 0x3c, 0x0a, 0x0b, 0x70, 0x6f, 0x64, 0x5f, 0x73, - 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x0a, 0x70, 0x6f, 0x64, 0x53, 0x75, - 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x6f, 0x64, 0x49, 0x70, 0x12, 0x28, 0x0a, 0x10, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x6f, - 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x5f, 0x73, 0x76, 0x63, 0x5f, 0x69, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, - 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x76, 0x63, 0x49, 0x70, 0x12, 0x2a, 0x0a, - 0x11, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x76, 0x63, 0x5f, 0x70, 0x6f, - 0x72, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x53, 0x76, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x6e, 0x6a, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x76, 0x63, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x0b, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x76, - 0x63, 0x48, 0x6f, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x07, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x6f, - 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x07, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x2b, - 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x03, 0x64, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0b, 0x6b, - 0x75, 0x62, 0x65, 0x5f, 0x64, 0x6e, 0x73, 0x5f, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x09, 0x6b, 0x75, 0x62, 0x65, 0x44, 0x6e, 0x73, 0x49, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x22, 0xfa, 0x01, 0x0a, 0x07, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x49, - 0x0a, 0x12, 0x61, 0x6c, 0x73, 0x6f, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x73, 0x75, 0x62, - 0x6e, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x10, 0x61, 0x6c, 0x73, 0x6f, 0x50, 0x72, 0x6f, - 0x78, 0x79, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x4b, 0x0a, 0x13, 0x6e, 0x65, 0x76, - 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, - 0x4e, 0x65, 0x74, 0x52, 0x11, 0x6e, 0x65, 0x76, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, - 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x57, 0x0a, 0x19, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x75, 0x62, 0x6e, - 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x73, 0x12, 0x56, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x09, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, + 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x5f, 0x0a, 0x0b, 0x65, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0x65, 0x0a, 0x0d, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x22, 0x65, 0x0a, 0x0f, 0x4c, 0x6f, + 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, + 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x22, 0x73, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x5f, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x74, 0x72, + 0x61, 0x66, 0x66, 0x69, 0x63, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, + 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x64, 0x5f, + 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x67, 0x65, 0x74, 0x50, + 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0xb7, 0x02, 0x0a, 0x0c, 0x4c, 0x6f, 0x67, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x6c, + 0x6f, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x17, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x43, 0x6f, 0x6e, - 0x66, 0x6c, 0x69, 0x63, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x22, - 0x9b, 0x01, 0x0a, 0x03, 0x44, 0x4e, 0x53, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x5f, 0x73, 0x75, 0x66, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, - 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x73, 0x75, - 0x66, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x78, - 0x63, 0x6c, 0x75, 0x64, 0x65, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x17, 0x0a, - 0x07, 0x6b, 0x75, 0x62, 0x65, 0x5f, 0x69, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, - 0x6b, 0x75, 0x62, 0x65, 0x49, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0x2c, 0x0a, - 0x09, 0x43, 0x4c, 0x49, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0x23, 0x0a, 0x0d, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x12, 0x0a, 0x05, - 0x66, 0x5f, 0x71, 0x5f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x66, 0x51, 0x4e, - 0x22, 0xc0, 0x01, 0x0a, 0x0c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, - 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, - 0x64, 0x5f, 0x69, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, - 0x70, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x69, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x07, 0x61, 0x70, 0x69, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x20, 0x0a, 0x0b, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0b, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x23, - 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4e, - 0x61, 0x6d, 0x65, 0x22, 0x52, 0x0a, 0x14, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, - 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x3a, 0x0a, 0x06, 0x61, - 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x65, 0x0a, 0x12, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, - 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, + 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x6f, + 0x64, 0x4c, 0x6f, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x4c, + 0x6f, 0x67, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x4a, 0x0a, 0x08, + 0x70, 0x6f, 0x64, 0x5f, 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x29, - 0x0a, 0x13, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x83, 0x01, 0x0a, 0x0d, 0x54, 0x75, - 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x63, - 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, - 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, - 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0b, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, - 0x53, 0x0a, 0x12, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, - 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x3d, 0x0a, 0x05, 0x6b, 0x69, 0x6e, 0x64, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, - 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x05, 0x6b, - 0x69, 0x6e, 0x64, 0x73, 0x22, 0x8d, 0x05, 0x0a, 0x0c, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, - 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, - 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b, 0x69, - 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x4e, 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x61, 0x67, 0x65, 0x6e, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x59, 0x0a, 0x11, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, - 0x65, 0x70, 0x74, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, - 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, - 0x10, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x73, 0x12, 0x3e, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, - 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x1a, 0x23, 0x0a, 0x09, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x22, 0x55, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0f, - 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, - 0x0e, 0x0a, 0x0a, 0x44, 0x45, 0x50, 0x4c, 0x4f, 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, - 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x53, 0x45, 0x54, 0x10, 0x02, 0x12, - 0x0f, 0x0a, 0x0b, 0x53, 0x54, 0x41, 0x54, 0x45, 0x46, 0x55, 0x4c, 0x53, 0x45, 0x54, 0x10, 0x03, - 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x10, 0x04, 0x22, 0x4d, 0x0a, - 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, - 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, - 0x0d, 0x0a, 0x09, 0x41, 0x56, 0x41, 0x49, 0x4c, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x0f, - 0x0a, 0x0b, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, - 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x03, 0x22, 0x46, 0x0a, 0x0a, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x4e, 0x4f, - 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x45, - 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x43, 0x45, 0x50, 0x54, - 0x45, 0x44, 0x10, 0x02, 0x22, 0xc7, 0x01, 0x0a, 0x0d, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, - 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, - 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, - 0x74, 0x79, 0x70, 0x65, 0x12, 0x3e, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, - 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, - 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x38, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x15, 0x0a, 0x11, - 0x41, 0x44, 0x44, 0x45, 0x44, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, - 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4d, 0x4f, 0x44, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, - 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0x84, - 0x01, 0x0a, 0x13, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x73, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xad, 0x01, 0x0a, 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, - 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x44, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x50, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x07, 0x70, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x1a, 0x3a, 0x0a, 0x0c, 0x50, 0x6f, 0x64, 0x4c, + 0x6f, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3a, 0x0a, 0x0c, 0x50, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x29, 0x0a, 0x13, 0x54, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x41, 0x50, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x3c, 0x0a, 0x0c, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x85, 0x01, 0x0a, 0x07, 0x4c, 0x69, + 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, + 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, + 0x6f, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x22, 0x6c, 0x0a, 0x15, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, + 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, + 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, + 0x72, 0x74, 0x12, 0x1e, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x63, 0x61, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x61, 0x88, + 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x63, 0x61, 0x22, + 0x3c, 0x0a, 0x19, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, + 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, + 0x63, 0x61, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0a, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x22, 0x29, 0x0a, + 0x0d, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, + 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x7c, 0x0a, 0x0b, 0x44, 0x69, 0x61, 0x6c, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x6e, 0x49, 0x64, + 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x74, 0x72, 0x69, 0x70, 0x5f, 0x6c, 0x61, + 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x72, 0x6f, 0x75, + 0x6e, 0x64, 0x74, 0x72, 0x69, 0x70, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x21, 0x0a, + 0x0c, 0x64, 0x69, 0x61, 0x6c, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0b, 0x64, 0x69, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, + 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x71, 0x0a, 0x0a, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x2a, 0xad, 0x01, 0x0a, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, - 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, - 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x01, 0x12, - 0x0b, 0x0a, 0x07, 0x57, 0x41, 0x49, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, - 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, 0x09, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, - 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x41, - 0x47, 0x45, 0x4e, 0x54, 0x10, 0x04, 0x12, 0x10, 0x0a, 0x0c, 0x4e, 0x4f, 0x5f, 0x4d, 0x45, 0x43, - 0x48, 0x41, 0x4e, 0x49, 0x53, 0x4d, 0x10, 0x05, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x50, - 0x4f, 0x52, 0x54, 0x53, 0x10, 0x06, 0x12, 0x0f, 0x0a, 0x0b, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, - 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x07, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x41, 0x44, 0x5f, 0x41, - 0x52, 0x47, 0x53, 0x10, 0x08, 0x32, 0xbc, 0x18, 0x0a, 0x07, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x12, 0x45, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, - 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x65, 0x0a, 0x0e, 0x47, 0x65, 0x74, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x2e, 0x74, 0x65, + 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x36, 0x0a, 0x0b, 0x44, 0x4e, 0x53, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x5f, 0x63, 0x6f, + 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, + 0x10, 0x0a, 0x03, 0x72, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x72, 0x72, + 0x73, 0x22, 0xca, 0x01, 0x0a, 0x10, 0x44, 0x4e, 0x53, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x3d, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2b, + 0x0a, 0x05, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x61, 0x73, 0x6b, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6d, 0x61, 0x73, 0x6b, 0x22, 0x8c, 0x04, 0x0a, 0x0b, + 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x42, 0x0a, 0x0e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, + 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x12, + 0x3c, 0x0a, 0x0b, 0x70, 0x6f, 0x64, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, + 0x74, 0x52, 0x0a, 0x70, 0x6f, 0x64, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, + 0x0e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x6f, + 0x64, 0x49, 0x70, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x70, + 0x6f, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x6f, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, + 0x0f, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x76, 0x63, 0x5f, 0x69, 0x70, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x53, 0x76, 0x63, 0x49, 0x70, 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x5f, 0x73, 0x76, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0f, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x76, 0x63, 0x50, 0x6f, 0x72, + 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x76, + 0x63, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, + 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x76, 0x63, 0x48, 0x6f, 0x73, 0x74, 0x12, 0x37, 0x0a, + 0x07, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x07, 0x72, + 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x2b, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x03, + 0x64, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0b, 0x6b, 0x75, 0x62, 0x65, 0x5f, 0x64, 0x6e, 0x73, 0x5f, + 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6b, 0x75, 0x62, 0x65, 0x44, 0x6e, + 0x73, 0x49, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x64, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0xfa, 0x01, 0x0a, 0x07, 0x52, + 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x49, 0x0a, 0x12, 0x61, 0x6c, 0x73, 0x6f, 0x5f, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, + 0x10, 0x61, 0x6c, 0x73, 0x6f, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, + 0x73, 0x12, 0x4b, 0x0a, 0x13, 0x6e, 0x65, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, + 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x11, 0x6e, 0x65, 0x76, + 0x65, 0x72, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x57, + 0x0a, 0x19, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, + 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x17, + 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x69, 0x6e, 0x67, + 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x22, 0x9b, 0x01, 0x0a, 0x03, 0x44, 0x4e, 0x53, 0x12, + 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x73, 0x75, 0x66, 0x66, 0x69, + 0x78, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x78, + 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x73, 0x75, 0x66, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x53, 0x75, 0x66, + 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x75, 0x62, 0x65, 0x5f, 0x69, 0x70, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6b, 0x75, 0x62, 0x65, 0x49, 0x70, 0x12, 0x25, + 0x0a, 0x0e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0x2c, 0x0a, 0x09, 0x43, 0x4c, 0x49, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x79, 0x61, 0x6d, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x59, + 0x61, 0x6d, 0x6c, 0x22, 0x23, 0x0a, 0x0d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, + 0x65, 0x46, 0x51, 0x4e, 0x12, 0x12, 0x0a, 0x05, 0x66, 0x5f, 0x71, 0x5f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x66, 0x51, 0x4e, 0x22, 0xc0, 0x01, 0x0a, 0x0c, 0x41, 0x67, 0x65, + 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x6f, 0x64, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x6f, 0x64, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x70, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x69, + 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x61, 0x70, 0x69, + 0x50, 0x6f, 0x72, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, + 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, + 0x61, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, + 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x52, 0x0a, 0x14, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x12, 0x3a, 0x0a, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, + 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x22, + 0x65, 0x0a, 0x12, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x29, 0x0a, 0x13, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x22, 0x83, 0x01, 0x0a, 0x0d, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, + 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, + 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x62, + 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x65, 0x67, 0x72, 0x65, + 0x73, 0x73, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x12, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, + 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x3d, 0x0a, + 0x05, 0x6b, 0x69, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, + 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x05, 0x6b, 0x69, 0x6e, 0x64, 0x73, 0x22, 0x8d, 0x05, 0x0a, + 0x0c, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, + 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x43, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x69, - 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x19, 0x43, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, - 0x75, 0x64, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x2f, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, - 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x0e, 0x47, - 0x65, 0x74, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, + 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, + 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, + 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x10, 0x0a, 0x03, + 0x75, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x4e, + 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, + 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x52, 0x0a, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x59, + 0x0a, 0x11, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x5f, 0x63, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x10, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, + 0x70, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3e, 0x0a, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, + 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x1a, 0x23, 0x0a, 0x09, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x22, 0x55, + 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x45, 0x50, 0x4c, 0x4f, + 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x50, 0x4c, 0x49, + 0x43, 0x41, 0x53, 0x45, 0x54, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x46, 0x55, 0x4c, 0x53, 0x45, 0x54, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x4f, 0x4c, 0x4c, + 0x4f, 0x55, 0x54, 0x10, 0x04, 0x22, 0x4d, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x17, + 0x0a, 0x13, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x56, 0x41, 0x49, 0x4c, + 0x41, 0x42, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, + 0x53, 0x53, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, + 0x52, 0x45, 0x10, 0x03, 0x22, 0x46, 0x0a, 0x0a, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x4e, 0x4f, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, + 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x49, + 0x4e, 0x54, 0x45, 0x52, 0x43, 0x45, 0x50, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0xc7, 0x01, 0x0a, + 0x0d, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3c, + 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x28, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3e, 0x0a, 0x08, + 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x38, 0x0a, 0x04, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x15, 0x0a, 0x11, 0x41, 0x44, 0x44, 0x45, 0x44, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4d, + 0x4f, 0x44, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, + 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0x84, 0x01, 0x0a, 0x13, 0x57, 0x6f, 0x72, 0x6b, 0x6c, + 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x30, + 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6d, 0x62, - 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x4a, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, + 0x12, 0x3b, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xad, 0x01, + 0x0a, 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x44, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x4c, 0x49, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x57, - 0x0a, 0x12, 0x47, 0x65, 0x74, 0x54, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x41, 0x50, 0x49, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x29, 0x2e, 0x74, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x30, 0x0a, + 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x12, + 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x2a, 0xad, 0x01, + 0x0a, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, 0x6f, + 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, + 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x57, 0x41, 0x49, 0x54, 0x49, + 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, + 0x09, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x03, + 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x10, 0x04, 0x12, 0x10, + 0x0a, 0x0c, 0x4e, 0x4f, 0x5f, 0x4d, 0x45, 0x43, 0x48, 0x41, 0x4e, 0x49, 0x53, 0x4d, 0x10, 0x05, + 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x50, 0x4f, 0x52, 0x54, 0x53, 0x10, 0x06, 0x12, 0x0f, + 0x0a, 0x0b, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x07, 0x12, + 0x0c, 0x0a, 0x08, 0x42, 0x41, 0x44, 0x5f, 0x41, 0x52, 0x47, 0x53, 0x10, 0x08, 0x32, 0xbc, 0x18, + 0x0a, 0x07, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x45, 0x0a, 0x07, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x41, 0x50, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x55, 0x0a, 0x0e, 0x41, 0x72, 0x72, 0x69, 0x76, - 0x65, 0x41, 0x73, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x53, - 0x0a, 0x0d, 0x41, 0x72, 0x72, 0x69, 0x76, 0x65, 0x41, 0x73, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, - 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, - 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x45, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x23, 0x2e, + 0x67, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x32, + 0x12, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, + 0x65, 0x46, 0x51, 0x4e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x23, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, + 0x4e, 0x12, 0x65, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x06, 0x44, 0x65, - 0x70, 0x61, 0x72, 0x74, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x4c, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x25, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, - 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x24, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4c, + 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x50, 0x6f, 0x64, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x2a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, - 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x5b, 0x0a, 0x0b, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, - 0x65, 0x6e, 0x74, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, + 0x19, 0x43, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x6d, 0x62, 0x61, 0x73, + 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x1a, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, + 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x2b, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, + 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4a, 0x0a, 0x0f, 0x47, 0x65, + 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x4c, 0x49, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x57, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x54, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x41, 0x50, 0x49, 0x12, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x41, 0x50, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x12, + 0x55, 0x0a, 0x0e, 0x41, 0x72, 0x72, 0x69, 0x76, 0x65, 0x41, 0x73, 0x43, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x53, 0x0a, 0x0d, 0x41, 0x72, 0x72, 0x69, 0x76, 0x65, + 0x41, 0x73, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x30, 0x01, 0x12, 0x5f, 0x0a, 0x0d, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x73, 0x4e, 0x53, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, - 0x74, 0x30, 0x01, 0x12, 0x63, 0x0a, 0x0f, 0x57, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x6a, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, - 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x12, 0x2b, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, - 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x44, 0x65, 0x6c, - 0x74, 0x61, 0x30, 0x01, 0x12, 0x5a, 0x0a, 0x10, 0x57, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x45, 0x0a, 0x06, 0x52, + 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, + 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x12, 0x43, 0x0a, 0x06, 0x44, 0x65, 0x70, 0x61, 0x72, 0x74, 0x12, 0x21, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4c, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, + 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, + 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, + 0x12, 0x24, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, + 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x57, 0x61, + 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x73, 0x12, 0x21, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, + 0x2a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, + 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x5b, 0x0a, + 0x0b, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x21, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, + 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, + 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x5f, 0x0a, 0x0d, 0x57, 0x61, + 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x4e, 0x53, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, - 0x12, 0x60, 0x0a, 0x0b, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, - 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, - 0x6f, 0x74, 0x12, 0x69, 0x0a, 0x10, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x74, - 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x70, - 0x61, 0x72, 0x65, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x64, 0x0a, - 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, + 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, + 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x63, 0x0a, 0x0f, 0x57, + 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x58, 0x0a, 0x0f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, - 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x32, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, - 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x1a, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, + 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, + 0x12, 0x6a, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, + 0x64, 0x73, 0x12, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, + 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x73, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x30, 0x01, 0x12, 0x5a, 0x0a, 0x10, + 0x57, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, + 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, 0x12, 0x60, 0x0a, 0x0b, 0x45, 0x6e, 0x73, 0x75, + 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x45, + 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, + 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x69, 0x0a, 0x10, 0x50, 0x72, + 0x65, 0x70, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x5e, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, - 0x65, 0x70, 0x74, 0x12, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x64, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x58, 0x0a, 0x0f, 0x52, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x57, 0x0a, 0x0f, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, - 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, - 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, 0x15, - 0x47, 0x65, 0x74, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, - 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0x1a, 0x16, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, - 0x64, 0x73, 0x12, 0x50, 0x0a, 0x09, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x12, - 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x16, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x6f, - 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x5e, 0x0a, 0x0c, 0x47, + 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x29, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x57, 0x0a, 0x0f, 0x52, + 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, - 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, - 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, - 0x6e, 0x66, 0x6f, 0x1a, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x12, 0x50, 0x0a, 0x0d, 0x57, 0x61, 0x74, 0x63, 0x68, - 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, - 0x1a, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x12, 0x56, 0x0a, 0x06, 0x54, 0x75, 0x6e, - 0x6e, 0x65, 0x6c, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, - 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, + 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x21, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, + 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x09, 0x4c, 0x6f, + 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, + 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x16, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, + 0x53, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, + 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, - 0x01, 0x12, 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, - 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x53, 0x0a, 0x09, 0x57, 0x61, 0x74, 0x63, 0x68, 0x44, 0x69, 0x61, 0x6c, 0x12, 0x21, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, - 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x30, 0x01, 0x42, 0x37, 0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x69, - 0x6f, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2f, 0x72, - 0x70, 0x63, 0x2f, 0x76, 0x32, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x20, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x12, + 0x50, 0x0a, 0x0d, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, + 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, + 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, + 0x01, 0x12, 0x56, 0x0a, 0x06, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x23, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, 0x09, 0x57, 0x61, 0x74, 0x63, 0x68, + 0x44, 0x69, 0x61, 0x6c, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, + 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x42, 0x37, 0x5a, 0x35, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x69, 0x6f, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x32, 0x2f, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, }) var ( diff --git a/rpc/manager/manager.proto b/rpc/manager/manager.proto index 51bbcae3c3..28a198a515 100644 --- a/rpc/manager/manager.proto +++ b/rpc/manager/manager.proto @@ -176,6 +176,9 @@ message InterceptSpec { // Whether to replace the running container. bool replace = 22; + + // Intercept desire no default port. + bool no_default_port = 25; } enum InterceptDispositionType { From a17fd20e1b8f6e328aa45be63768d09b0e238cdf Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Mon, 13 Jan 2025 08:48:28 +0100 Subject: [PATCH 18/61] Integration tests for the replace command. Signed-off-by: Thomas Hallgren --- integration_test/helm_test.go | 4 +- integration_test/intercept_localhost_test.go | 2 +- integration_test/itest/cluster.go | 12 +- integration_test/manual_agent_test.go | 2 +- integration_test/multiple_services_test.go | 2 +- integration_test/replace_test.go | 125 ++++++++++++++++++ integration_test/uninstall_test.go | 2 +- integration_test/webhook_test.go | 2 +- .../workload_configuration_test.go | 4 +- 9 files changed, 140 insertions(+), 15 deletions(-) create mode 100644 integration_test/replace_test.go diff --git a/integration_test/helm_test.go b/integration_test/helm_test.go index c0f59f5ca6..c95411dd45 100644 --- a/integration_test/helm_test.go +++ b/integration_test/helm_test.go @@ -71,7 +71,7 @@ func (s *helmSuite) Test_HelmWebhookInjectsInManagedNamespace() { s.Eventually(func() bool { stdout, _, err := itest.Telepresence(ctx, "list", "--agents") - return err == nil && strings.Contains(stdout, "echo-auto-inject: ready to intercept (traffic-agent already installed)") + return err == nil && strings.Contains(stdout, "echo-auto-inject: ready to engage (traffic-agent already installed)") }, 20*time.Second, // waitFor 2*time.Second, // polling interval @@ -85,7 +85,7 @@ func (s *helmSuite) Test_HelmWebhookDoesntInjectInUnmanagedNamespace() { s.Never(func() bool { stdout, _, err := itest.Telepresence(ctx, "list", "--namespace", s.appSpace2, "--agents") - return err == nil && strings.Contains(stdout, "echo-auto-inject: ready to intercept (traffic-agent already installed)") + return err == nil && strings.Contains(stdout, "echo-auto-inject: ready to engage (traffic-agent already installed)") }, 10*time.Second, // waitFor 2*time.Second, // polling interval diff --git a/integration_test/intercept_localhost_test.go b/integration_test/intercept_localhost_test.go index ece7efdad4..f6a87dad2a 100644 --- a/integration_test/intercept_localhost_test.go +++ b/integration_test/intercept_localhost_test.go @@ -39,7 +39,7 @@ func (s *interceptLocalhostSuite) SetupSuite() { s.defaultRoute, err = routing.DefaultRoute(ctx) s.Require().NoError(err) dlog.Infof(ctx, "ip: %s: route: %s", s.defaultRoute.LocalIP, s.defaultRoute) - s.port, s.cancelLocal = itest.StartLocalHttpEchoServerWithHost(ctx, s.ServiceName(), s.defaultRoute.LocalIP.String()) + s.port, s.cancelLocal = itest.StartLocalHttpEchoServerWithAddr(ctx, s.ServiceName(), net.JoinHostPort(s.defaultRoute.LocalIP.String(), "0")) s.CapturePodLogs(ctx, "echo", "traffic-agent", s.AppNamespace()) } diff --git a/integration_test/itest/cluster.go b/integration_test/itest/cluster.go index c40a72504f..9a0993ff5f 100644 --- a/integration_test/itest/cluster.go +++ b/integration_test/itest/cluster.go @@ -1120,11 +1120,11 @@ func DeleteNamespaces(ctx context.Context, namespaces ...string) { wg.Wait() } -// StartLocalHttpEchoServerWithHost is like StartLocalHttpEchoServer but binds to a specific host instead of localhost. -func StartLocalHttpEchoServerWithHost(ctx context.Context, name string, host string) (int, context.CancelFunc) { +// StartLocalHttpEchoServerWithAddr is like StartLocalHttpEchoServer but binds to a specific host instead of localhost. +func StartLocalHttpEchoServerWithAddr(ctx context.Context, name, addr string) (int, context.CancelFunc) { ctx, cancel := context.WithCancel(ctx) lc := net.ListenConfig{} - l, err := lc.Listen(ctx, "tcp", net.JoinHostPort(host, "0")) + l, err := lc.Listen(ctx, "tcp", addr) require.NoError(getT(ctx), err, "failed to listen on localhost") sc := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1140,9 +1140,9 @@ func StartLocalHttpEchoServerWithHost(ctx context.Context, name string, host str defer cancel() err = sc.Shutdown(ctx) if err != nil { - dlog.Errorf(ctx, "http server on %s exited with error: %v", host, err) + dlog.Errorf(ctx, "http server on %s exited with error: %v", addr, err) } else { - dlog.Errorf(ctx, "http server on %s exited", host) + dlog.Errorf(ctx, "http server on %s exited", addr) } }() return l.Addr().(*net.TCPAddr).Port, cancel @@ -1151,7 +1151,7 @@ func StartLocalHttpEchoServerWithHost(ctx context.Context, name string, host str // StartLocalHttpEchoServer starts a local http server that echoes a line with the given name and // the current URL path. The port is returned together with function that cancels the server. func StartLocalHttpEchoServer(ctx context.Context, name string) (int, context.CancelFunc) { - return StartLocalHttpEchoServerWithHost(ctx, name, "localhost") + return StartLocalHttpEchoServerWithAddr(ctx, name, "localhost:0") } // PingInterceptedEchoServer assumes that a server has been created using StartLocalHttpEchoServer and diff --git a/integration_test/manual_agent_test.go b/integration_test/manual_agent_test.go index daf2991c91..48bc019fba 100644 --- a/integration_test/manual_agent_test.go +++ b/integration_test/manual_agent_test.go @@ -145,7 +145,7 @@ func testManualAgent(s *itest.Suite, nsp itest.NamespacePair) { defer itest.TelepresenceQuitOk(ctx) stdout = itest.TelepresenceOk(ctx, "list") - require.Regexp(regexp.MustCompile(`.*`+ac.WorkloadName+`\s*:\s*ready to intercept \(traffic-agent already installed\).*`), stdout) + require.Regexp(regexp.MustCompile(`.*`+ac.WorkloadName+`\s*:\s*ready to engage \(traffic-agent already installed\).*`), stdout) svcPort, svcCancel := itest.StartLocalHttpEchoServer(ctx, ac.WorkloadName) defer svcCancel() diff --git a/integration_test/multiple_services_test.go b/integration_test/multiple_services_test.go index cb01666982..3e635a730e 100644 --- a/integration_test/multiple_services_test.go +++ b/integration_test/multiple_services_test.go @@ -95,7 +95,7 @@ func (s *multipleServicesSuite) Test_LargeRequest() { func (s *multipleServicesSuite) Test_List() { stdout := itest.TelepresenceOk(s.Context(), "-n", s.AppNamespace(), "list") for i := 0; i < s.ServiceCount(); i++ { - s.Regexp(fmt.Sprintf(`%s-%d\s*: ready to intercept`, s.Name(), i), stdout) + s.Regexp(fmt.Sprintf(`%s-%d\s*: ready to engage`, s.Name(), i), stdout) } } diff --git a/integration_test/replace_test.go b/integration_test/replace_test.go new file mode 100644 index 0000000000..3bbfd7ea5c --- /dev/null +++ b/integration_test/replace_test.go @@ -0,0 +1,125 @@ +package integration_test + +import ( + "fmt" + "path/filepath" + "time" + + core "k8s.io/api/core/v1" + + "github.com/datawire/dlib/dlog" + "github.com/telepresenceio/telepresence/v2/integration_test/itest" + "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" +) + +type replaceSuite struct { + itest.Suite + itest.TrafficManager + svc string + tplPath string + tpl *itest.Generic +} + +func (s *replaceSuite) SuiteName() string { + return "Replace" +} + +func init() { + itest.AddConnectedSuite("", func(h itest.TrafficManager) itest.TestingSuite { + return &replaceSuite{Suite: itest.Suite{Harness: h}, TrafficManager: h, svc: "echo-server"} + }) +} + +func (s *replaceSuite) SetupSuite() { + s.Suite.SetupSuite() + s.tplPath = filepath.Join("testdata", "k8s", "generic.goyaml") + s.tpl = &itest.Generic{ + Name: s.svc, + TargetPort: "http", + Registry: "ghcr.io/telepresenceio", + Image: "echo-server:latest", + ServicePorts: []itest.ServicePort{ + { + Number: 80, + Name: "http", + TargetPort: "http-cp", + }, + { + Number: 81, + Name: "extra", + TargetPort: "extra-cp", + }, + }, + ContainerPorts: []itest.ContainerPort{ + { + Number: 8080, + Name: "http-cp", + }, + { + Number: 8081, + Name: "extra-cp", + }, + }, + Environment: []core.EnvVar{ + { + Name: "PORTS", + Value: "8080,8081", + }, + { + Name: "LISTEN_ADDRESS", + ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + }, + Annotations: map[string]string{ + agentconfig.InjectAnnotation: "enabled", + }, + } + ctx := s.Context() + s.ApplyTemplate(ctx, s.tplPath, s.tpl) + s.NoError(s.RolloutStatusWait(ctx, "deploy/"+s.svc)) +} + +func (s *replaceSuite) TearDownSuite() { + s.DeleteTemplate(s.Context(), s.tplPath, s.tpl) +} + +func (s *replaceSuite) Test_ReplaceWithMultiContainerPorts() { + ctx := s.Context() + _, httpCancel := itest.StartLocalHttpEchoServerWithAddr(ctx, s.svc+"-http", "localhost:8080") + defer httpCancel() + _, extraCancel := itest.StartLocalHttpEchoServerWithAddr(ctx, s.svc+"-extra", "localhost:8081") + defer extraCancel() + + // Use container port names here. + so := itest.TelepresenceOk(ctx, "replace", s.svc) + mustLeave := true + defer func() { + if mustLeave { + itest.TelepresenceOk(ctx, "leave", s.svc) + } + }() + + rq := s.Require() + rq.Contains(so, "Using Deployment "+s.svc) + + itest.PingInterceptedEchoServer(ctx, fmt.Sprintf("%s/%s-http", s.svc, s.svc), "80") + itest.PingInterceptedEchoServer(ctx, fmt.Sprintf("%s/%s-extra", s.svc, s.svc), "81") + + itest.TelepresenceOk(ctx, "leave", s.svc) + mustLeave = false + + // Ensure that we now reach the original app again. + s.Eventually(func() bool { + out, err := itest.Output(ctx, "curl", "--verbose", "--max-time", "1", s.svc) + dlog.Infof(ctx, "Received %s", out) + if err != nil { + dlog.Errorf(ctx, "curl error %v", err) + return false + } + return true + }, 30*time.Second, 2*time.Second, "Pod app is not reachable after ending the replace") +} diff --git a/integration_test/uninstall_test.go b/integration_test/uninstall_test.go index abddc0270a..4ae50335a8 100644 --- a/integration_test/uninstall_test.go +++ b/integration_test/uninstall_test.go @@ -34,7 +34,7 @@ func (s *notConnectedSuite) Test_Uninstall() { s.Eventually(func() bool { stdout, _, err = itest.Telepresence(ctx, "list", "--agents") - return err == nil && strings.Contains(stdout, jobname+": ready to intercept (traffic-agent already installed)") + return err == nil && strings.Contains(stdout, jobname+": ready to engage (traffic-agent already installed)") }, 30*time.Second, 3*time.Second) stdout = itest.TelepresenceOk(ctx, "helm", "uninstall", "-n", s.ManagerNamespace()) diff --git a/integration_test/webhook_test.go b/integration_test/webhook_test.go index 6f21ff78a1..94c5e5da07 100644 --- a/integration_test/webhook_test.go +++ b/integration_test/webhook_test.go @@ -31,7 +31,7 @@ func (s *webhookSuite) Test_AutoInjectedAgent() { require := s.Require() require.Eventually(func() bool { stdout, _, err := itest.Telepresence(ctx, "list", "--agents") - return err == nil && strings.Contains(stdout, "echo-auto-inject: ready to intercept (traffic-agent already installed)") + return err == nil && strings.Contains(stdout, "echo-auto-inject: ready to engage (traffic-agent already installed)") }, 20*time.Second, // waitFor 2*time.Second, // polling interval diff --git a/integration_test/workload_configuration_test.go b/integration_test/workload_configuration_test.go index 0e4e050ab9..b7ccd2be1b 100644 --- a/integration_test/workload_configuration_test.go +++ b/integration_test/workload_configuration_test.go @@ -85,7 +85,7 @@ func (s *workloadConfigurationSuite) Test_InterceptsDeploymentWithDisabledReplic require.Eventually( func() bool { stdout, _, err := itest.Telepresence(ctx, "list") - return err == nil && strings.Contains(stdout, fmt.Sprintf("%s: ready to intercept", wl)) + return err == nil && strings.Contains(stdout, fmt.Sprintf("%s: ready to engage", wl)) }, 6*time.Second, // waitFor 2*time.Second, // polling interval @@ -120,7 +120,7 @@ func (s *workloadConfigurationSuite) Test_InterceptsReplicaSetWithDisabledDeploy require.Eventually( func() bool { stdout, _, err := itest.Telepresence(ctx, "list") - return err == nil && strings.Contains(stdout, fmt.Sprintf("%s: ready to intercept", interceptableWl)) + return err == nil && strings.Contains(stdout, fmt.Sprintf("%s: ready to engage", interceptableWl)) }, 6*time.Second, // waitFor 2*time.Second, // polling interval From e9608313d03bfca9b3dff479c8630dd6783be1cd Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 8 Feb 2025 15:46:21 +0100 Subject: [PATCH 19/61] List output includes workload kind. The output of the `telepresence list` command will now include the workload kind (deployment, replicaset, statefulset, or rollout) in all entries. Signed-off-by: Thomas Hallgren --- CHANGELOG.yml | 5 +++++ docs/release-notes.md | 6 ++++++ docs/release-notes.mdx | 4 ++++ pkg/client/cli/cmd/list.go | 13 +++++++++++-- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.yml b/CHANGELOG.yml index acc8104bd5..e5a825ce10 100644 --- a/CHANGELOG.yml +++ b/CHANGELOG.yml @@ -85,6 +85,11 @@ items: values: `. ``` docs: install/manager#static-versus-dynamic-namespace-selection + - type: feature + title: List output includes workload kind. + body: >- + The output of the `telepresence list` command will now include the workload kind (deployment, replicaset, + statefulset, or rollout) in all entries. - type: change title: Drop deprecated current-cluster-id command. body: >- diff --git a/docs/release-notes.md b/docs/release-notes.md index 44e37643d2..e96ecda404 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -66,6 +66,12 @@ namespaceSelector: ```
+##
feature
List output includes workload kind.
+
+ +The output of the `telepresence list` command will now include the workload kind (deployment, replicaset, statefulset, or rollout) in all entries. +
+ ##
change
Drop deprecated current-cluster-id command.
diff --git a/docs/release-notes.mdx b/docs/release-notes.mdx index a1ac700c7a..6fe709ff5c 100644 --- a/docs/release-notes.mdx +++ b/docs/release-notes.mdx @@ -64,6 +64,10 @@ namespaceSelector: values: `. ``` + + List output includes workload kind. + The output of the `telepresence list` command will now include the workload kind (deployment, replicaset, statefulset, or rollout) in all entries. + Drop deprecated current-cluster-id command. The clusterID was deprecated some time ago, and replaced by the ID of the namespace where the traffic-manager is installed. diff --git a/pkg/client/cli/cmd/list.go b/pkg/client/cli/cmd/list.go index c5dc3f2ea1..0fc1e319fb 100644 --- a/pkg/client/cli/cmd/list.go +++ b/pkg/client/cli/cmd/list.go @@ -3,6 +3,7 @@ package cmd import ( "context" "io" + "strings" "github.com/spf13/cobra" "google.golang.org/grpc" @@ -210,10 +211,17 @@ func (s *listCommand) printList(ctx context.Context, workloads []*connector.Work } ns = depNs } + typeLen := 0 + nameLen := 0 for _, dep := range workloads { - n := dep.Name + n := dep.WorkloadResourceType nl := len(n) + if nl > typeLen { + typeLen = nl + } + n = dep.Name + nl = len(n) if includeNs { nl += len(dep.Namespace) + 1 } @@ -222,11 +230,12 @@ func (s *listCommand) printList(ctx context.Context, workloads []*connector.Work } } for _, workload := range workloads { + t := workload.WorkloadResourceType n := workload.Name if includeNs { n += "." + workload.Namespace } - ioutil.Printf(stdout, "%-*s: %s\n", nameLen, n, state(workload)) + ioutil.Printf(stdout, "%-*s %-*s: %s\n", typeLen, strings.ToLower(t), nameLen, n, state(workload)) } } } From 9009905e252e356d9ccce0c97017543f796ea8b0 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 1 Feb 2025 12:52:14 +0100 Subject: [PATCH 20/61] Drop use of the telepresence-agents configmap. This commit removes the complex pattern used when engaging with a workload to get a traffic-agent injected into its pods. 1. Generate an agent configuration. 2. Store the configuration in the `telepresence-agents` configmap. 3. Wait for the agent to arrive. A watcher on the `telepresence-agents` configmap would now get an event and discover the new or modified configuration. It took the following steps: 1. Determine if a rollout is needed. 2. Perform a rollout to trigger the mutating webhook. 1. Check if the pod has a configuration entry in the configmap. 2. Patch pod if necessary. 1. Generate an agent configuration. 2. Evict pods with config annotation mismatch (triggers the webhook). No action. The configmap no longer exists 1. Check if the pod has a configuration entry in the configmap. 2. Patch pod if necessary and include the config as an annotation. Signed-off-by: Thomas Hallgren --- CHANGELOG.yml | 12 + .../templates/agent-configmap.yaml | 9 - .../trafficManagerRbac/cluster-scope.yaml | 26 +- .../trafficManagerRbac/namespace-scope.yaml | 24 +- cmd/traffic/cmd/agent/agent.go | 22 +- cmd/traffic/cmd/agent/agent_test.go | 12 +- cmd/traffic/cmd/agent/config.go | 10 +- cmd/traffic/cmd/agent/containerstate.go | 6 +- cmd/traffic/cmd/agent/state.go | 2 +- cmd/traffic/cmd/agentinit/agent_init.go | 16 +- .../cmd/manager/mutator/agent_injector.go | 54 +- .../manager/mutator/agent_injector_test.go | 221 +++-- .../cmd/manager/mutator/configmap_watcher.go | 131 --- cmd/traffic/cmd/manager/mutator/constants.go | 6 +- .../cmd/manager/mutator/service_watcher.go | 42 +- cmd/traffic/cmd/manager/mutator/watcher.go | 873 ++++++------------ .../cmd/manager/mutator/workload_watcher.go | 44 +- cmd/traffic/cmd/manager/service.go | 41 +- cmd/traffic/cmd/manager/service_test.go | 93 +- cmd/traffic/cmd/manager/state/intercept.go | 335 +++---- .../cmd/manager/state/presence_test.go | 5 +- cmd/traffic/cmd/manager/state/session.go | 7 - cmd/traffic/cmd/manager/state/state.go | 277 +++--- cmd/traffic/cmd/manager/state/state_test.go | 80 +- .../manager/state/workload_info_watcher.go | 12 +- .../cmd/manager/test/testdata/agents.yaml | 4 + cmd/traffic/cmd/manager/watchable/map.go | 546 ++++------- cmd/traffic/cmd/manager/watchable/map_test.go | 426 --------- .../cmd/manager/watchable/util_test.go | 26 - docs/release-notes.md | 12 + docs/release-notes.mdx | 8 + integration_test/injector_test.go | 79 +- integration_test/manual_agent_test.go | 47 +- integration_test/multiport_test.go | 4 +- .../workload_configuration_test.go | 14 - pkg/agentconfig/container.go | 61 +- pkg/agentconfig/container_test.go | 29 - pkg/agentconfig/sidecar.go | 106 ++- pkg/client/cli/cmd/genyaml.go | 59 +- pkg/client/userd/trafficmgr/session.go | 18 +- pkg/workload/util.go | 9 - pkg/workload/watcher.go | 3 +- rpc/manager/manager.pb.go | 704 +++++++------- rpc/manager/manager.proto | 13 + rpc/manager/manager_grpc.pb.go | 42 + 45 files changed, 1764 insertions(+), 2806 deletions(-) delete mode 100644 charts/telepresence/templates/agent-configmap.yaml delete mode 100644 cmd/traffic/cmd/manager/mutator/configmap_watcher.go delete mode 100644 cmd/traffic/cmd/manager/watchable/map_test.go delete mode 100644 cmd/traffic/cmd/manager/watchable/util_test.go diff --git a/CHANGELOG.yml b/CHANGELOG.yml index e5a825ce10..09c7359f7b 100644 --- a/CHANGELOG.yml +++ b/CHANGELOG.yml @@ -90,6 +90,18 @@ items: body: >- The output of the `telepresence list` command will now include the workload kind (deployment, replicaset, statefulset, or rollout) in all entries. + - type: change + title: Trigger the mutating webhook with Kubernetes eviction objects instead of patching workloads. + body: >- + Instead of patching workloads, or scaling the workloads down to zero and up again, Telepresence will now + create policy/v1 Eviction objects to trigger the mutating webhook. This causes a slight change in the + traffic-manager RBAC. The `patch` permissions are no longer needed. Instead, the traffic-manager must be + able to create "pod/eviction" objects. + - type: change + title: The telepresence-agents configmap is no longer used. + body: >- + The traffic-agent configuration was moved into a pod-annotation. This avoids sync problems between the + telepresence-agents (which is no no longer present) and the pods. - type: change title: Drop deprecated current-cluster-id command. body: >- diff --git a/charts/telepresence/templates/agent-configmap.yaml b/charts/telepresence/templates/agent-configmap.yaml deleted file mode 100644 index 380290467a..0000000000 --- a/charts/telepresence/templates/agent-configmap.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- if .Values.agentInjector.enabled }} -{{- if and (not .Values.rbac.only) .Values.agentInjector.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: telepresence-agents - namespace: {{ include "traffic-manager.namespace" $ }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/telepresence/templates/trafficManagerRbac/cluster-scope.yaml b/charts/telepresence/templates/trafficManagerRbac/cluster-scope.yaml index c9b1c40e66..c565ab9146 100644 --- a/charts/telepresence/templates/trafficManagerRbac/cluster-scope.yaml +++ b/charts/telepresence/templates/trafficManagerRbac/cluster-scope.yaml @@ -25,41 +25,27 @@ rules: - "" resources: - nodes - - pods - services - namespaces + - pods verbs: - list - get - watch - apiGroups: - "" - resources: - - pods/log - verbs: - - get {{- if .agentInjector.enabled }} -- apiGroups: - - "" resources: - - configmaps + - pods/eviction verbs: - create {{- end }} - apiGroups: - "" resources: - - configmaps + - pods/log verbs: - - list - get - - watch -{{- if .agentInjector.enabled }} - - update - - delete -{{- end }} - resourceNames: - - telepresence-agents - apiGroups: - "apps" resources: @@ -70,9 +56,6 @@ rules: - get - list - watch -{{- if .agentInjector.enabled }} - - patch -{{- end }} {{- if .workloads.argoRollouts.enabled }} - apiGroups: - "argoproj.io" @@ -82,9 +65,6 @@ rules: - get - list - watch -{{- if .agentInjector.enabled }} - - patch -{{- end }} {{- end }} - apiGroups: - "events.k8s.io" diff --git a/charts/telepresence/templates/trafficManagerRbac/namespace-scope.yaml b/charts/telepresence/templates/trafficManagerRbac/namespace-scope.yaml index 2c5327db17..5189e9eb69 100644 --- a/charts/telepresence/templates/trafficManagerRbac/namespace-scope.yaml +++ b/charts/telepresence/templates/trafficManagerRbac/namespace-scope.yaml @@ -31,26 +31,26 @@ rules: - apiGroups: - "" resources: - - pods - services + - pods verbs: - list - get - watch +{{- if $interceptEnabled }} - apiGroups: - "" resources: - - pods/log + - pods/eviction verbs: - - get -{{- if $interceptEnabled }} + - create +{{- end }} - apiGroups: - "" resources: - - configmaps + - pods/log verbs: - - create -{{- end }} + - get - apiGroups: - "" resources: @@ -59,17 +59,10 @@ rules: - list - get - watch -{{- if $interceptEnabled }} - - update - - delete -{{- end }} resourceNames: {{- if eq . $managerNamespace }} - {{ include "traffic-manager.name" $ }} {{- end }} -{{- if has . $namespaces }} - - telepresence-agents -{{- end }} - apiGroups: - "apps" resources: @@ -80,9 +73,6 @@ rules: - get - list - watch -{{- if $interceptEnabled }} - - patch -{{- end }} {{- if $argoRolloutsEnabled }} - apiGroups: - "argoproj.io" diff --git a/cmd/traffic/cmd/agent/agent.go b/cmd/traffic/cmd/agent/agent.go index 22a7431c51..500a70d637 100644 --- a/cmd/traffic/cmd/agent/agent.go +++ b/cmd/traffic/cmd/agent/agent.go @@ -13,6 +13,8 @@ import ( "github.com/pkg/sftp" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/datawire/dlib/dgroup" "github.com/datawire/dlib/dhttp" @@ -41,9 +43,10 @@ func AppEnvironment(ctx context.Context, ag *agentconfig.Container) (map[string] // Keys that aren't useful when running on the local machine. skipKeys := map[string]bool{ - "HOME": true, - "PATH": true, - "HOSTNAME": true, + "HOME": true, + "PATH": true, + "HOSTNAME": true, + agentconfig.EnvAgentConfig: true, } // Add prefixed variables separately last, so that we can @@ -131,7 +134,7 @@ func Main(ctx context.Context, _ ...string) error { // Handle configuration config, err := LoadConfig(ctx) if err != nil { - return err + return fmt.Errorf("unable to load config: %w", err) } g := dgroup.NewGroup(ctx, dgroup.GroupConfig{ @@ -164,7 +167,7 @@ func sidecar(ctx context.Context, s State, info *rpc.AgentInfo) error { icStates := make(map[agentconfig.PortAndProto][]*agentconfig.Intercept, len(cn.Intercepts)) for _, ic := range cn.Intercepts { ap := ic.AgentPort - if cn.Replace { + if cn.Replace == agentconfig.ReplacePolicyContainer { // Listen to replaced container's original port. ap = ic.ContainerPort } @@ -176,7 +179,7 @@ func sidecar(ctx context.Context, s State, info *rpc.AgentInfo) error { ic := ics[0] // They all have the same protocol container port, so the first one will do. var fwd forwarder.Interceptor var cp uint16 - if !cn.Replace { + if cn.Replace == agentconfig.ReplacePolicyIntercept { if ic.TargetPortNumeric { // We must differentiate between connections originating from the agent's forwarder to the container // port and those from other sources. The former should not be routed back, while the latter should @@ -215,7 +218,12 @@ func TalkToManagerLoop(ctx context.Context, s State, info *rpc.AgentInfo) { for { if err := TalkToManager(ctx, gRPCAddress, info, s); err != nil { - dlog.Info(ctx, err) + switch status.Code(err) { + case codes.AlreadyExists, codes.Aborted: + // This won't change, so abort here. + return + } + dlog.Error(ctx, err) } select { diff --git a/cmd/traffic/cmd/agent/agent_test.go b/cmd/traffic/cmd/agent/agent_test.go index 5eb04bc750..c466f0920d 100644 --- a/cmd/traffic/cmd/agent/agent_test.go +++ b/cmd/traffic/cmd/agent/agent_test.go @@ -2,14 +2,12 @@ package agent_test import ( "context" - "path/filepath" "runtime" "testing" "github.com/spf13/afero" "github.com/stretchr/testify/require" core "k8s.io/api/core/v1" - "sigs.k8s.io/yaml" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/agent" @@ -60,14 +58,13 @@ func testContext(t *testing.T, env dos.MapEnv) context.Context { if env == nil { env = make(dos.MapEnv) } - y, err := yaml.Marshal(&testConfig) - require.NoError(t, err) - - require.NoError(t, fs.MkdirAll(agentconfig.ConfigMountPoint, 0o700)) require.NoError(t, fs.MkdirAll(agentconfig.ExportsMountPoint, 0o700)) - require.NoError(t, afero.WriteFile(fs, filepath.Join(agentconfig.ConfigMountPoint, agentconfig.ConfigFile), y, 0o600)) + + cfgJSON, err := agentconfig.MarshalTight(&testConfig) + require.NoError(t, err) env[agentconfig.EnvPrefixAgent+"POD_IP"] = podIP + env[agentconfig.EnvAgentConfig] = cfgJSON ctx := dlog.NewTestContext(t, false) ctx = dos.WithFS(ctx, aferofs.Wrap(fs)) @@ -78,6 +75,7 @@ func Test_LoadConfig(t *testing.T) { ctx := testContext(t, nil) config, err := agent.LoadConfig(ctx) require.NoError(t, err) + testConfig.AgentImage = "" require.Equal(t, &testConfig, config.AgentConfig()) require.Equal(t, podIP, config.PodIP()) } diff --git a/cmd/traffic/cmd/agent/config.go b/cmd/traffic/cmd/agent/config.go index f6f9fde5da..c5aac7ff7a 100644 --- a/cmd/traffic/cmd/agent/config.go +++ b/cmd/traffic/cmd/agent/config.go @@ -3,6 +3,7 @@ package agent import ( "bufio" "context" + "errors" "fmt" "os" "path/filepath" @@ -31,13 +32,14 @@ type config struct { } func LoadConfig(ctx context.Context) (Config, error) { - bs, err := dos.ReadFile(ctx, filepath.Join(agentconfig.ConfigMountPoint, agentconfig.ConfigFile)) - if err != nil { - return nil, fmt.Errorf("unable to open agent ConfigMap: %w", err) + cfgTight, ok := dos.LookupEnv(ctx, agentconfig.EnvAgentConfig) + if !ok { + return nil, errors.New("unable to retrieve agent ConfigMap entry") } + var err error c := config{} - c.sidecarExt, err = agentconfig.UnmarshalYAML(bs) + c.sidecarExt, err = agentconfig.UnmarshalJSON(cfgTight) if err != nil { return nil, fmt.Errorf("unable to decode agent ConfigMap: %w", err) } diff --git a/cmd/traffic/cmd/agent/containerstate.go b/cmd/traffic/cmd/agent/containerstate.go index 597945586a..109bd19495 100644 --- a/cmd/traffic/cmd/agent/containerstate.go +++ b/cmd/traffic/cmd/agent/containerstate.go @@ -27,8 +27,8 @@ func (c *containerState) Name() string { return c.container.Name } -func (c *containerState) Replace() bool { - return bool(c.container.Replace) +func (c *containerState) ReplaceContainer() bool { + return c.container.Replace == agentconfig.ReplacePolicyContainer } // HandleIntercepts on the containerState takes care of intercepts that just replaces a container and do not declare @@ -37,7 +37,7 @@ func (c *containerState) HandleIntercepts(ctx context.Context, iis []*manager.In for _, ii := range iis { if ii.Disposition == manager.InterceptDispositionType_WAITING { spec := ii.Spec - if c.Replace() && c.Name() == spec.ContainerName && spec.ContainerPort == 0 { + if c.ReplaceContainer() && c.Name() == spec.ContainerName && spec.ContainerPort == 0 { dlog.Debugf(ctx, "container %s handling replace %s", c.Name(), spec.Name) rs = append(rs, &manager.ReviewInterceptRequest{ Id: ii.Id, diff --git a/cmd/traffic/cmd/agent/state.go b/cmd/traffic/cmd/agent/state.go index 9db5994e4c..3a7b0ab93f 100644 --- a/cmd/traffic/cmd/agent/state.go +++ b/cmd/traffic/cmd/agent/state.go @@ -39,7 +39,7 @@ type State interface { type ContainerState interface { State Name() string - Replace() bool + ReplaceContainer() bool MountPoint() string Env() map[string]string } diff --git a/cmd/traffic/cmd/agentinit/agent_init.go b/cmd/traffic/cmd/agentinit/agent_init.go index 22d7e68375..3230e4fa49 100644 --- a/cmd/traffic/cmd/agentinit/agent_init.go +++ b/cmd/traffic/cmd/agentinit/agent_init.go @@ -5,11 +5,11 @@ package agentinit import ( "context" + "errors" "fmt" "net" "net/netip" "os" - "path/filepath" "strconv" "strings" @@ -19,7 +19,6 @@ import ( "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" - "github.com/telepresenceio/telepresence/v2/pkg/dos" "github.com/telepresenceio/telepresence/v2/pkg/version" ) @@ -31,14 +30,15 @@ type config struct { agentconfig.SidecarExt } -func loadConfig(ctx context.Context) (*config, error) { - bs, err := dos.ReadFile(ctx, filepath.Join(agentconfig.ConfigMountPoint, agentconfig.ConfigFile)) - if err != nil { - return nil, fmt.Errorf("unable to open agent ConfigMap: %w", err) +func loadConfig() (*config, error) { + cfgTight, ok := os.LookupEnv(agentconfig.EnvAgentConfig) + if !ok { + return nil, errors.New("unable to retrieve agent ConfigMap entry") } c := config{} - c.SidecarExt, err = agentconfig.UnmarshalYAML(bs) + var err error + c.SidecarExt, err = agentconfig.UnmarshalJSON(cfgTight) if err != nil { return nil, fmt.Errorf("unable to decode agent ConfigMap: %w", err) } @@ -201,7 +201,7 @@ func Main(ctx context.Context, args ...string) error { dlog.Error(ctx, derror.PanicToError(r)) } }() - cfg, err := loadConfig(ctx) + cfg, err := loadConfig() if err != nil { dlog.Error(ctx, err) return err diff --git a/cmd/traffic/cmd/manager/mutator/agent_injector.go b/cmd/traffic/cmd/manager/mutator/agent_injector.go index 7ca889b326..fe3525e65c 100644 --- a/cmd/traffic/cmd/manager/mutator/agent_injector.go +++ b/cmd/traffic/cmd/manager/mutator/agent_injector.go @@ -110,7 +110,7 @@ func (a *agentInjector) Inject(ctx context.Context, req *admission.AdmissionRequ } if isDelete { - a.agentConfigs.Blacklist(pod.Name, pod.Namespace) + a.agentConfigs.Inactivate(pod.Name) return nil, nil } @@ -166,22 +166,10 @@ func (a *agentInjector) Inject(ctx context.Context, req *admission.AdmissionRequ // Not an error. It just means that the pod is not eligible for intercepts. return nil, nil } - scx, err = a.agentConfigs.Get(ctx, wl.GetName(), wl.GetNamespace()) + scx = a.agentConfigs.Get(wl.GetName(), wl.GetNamespace()) switch { - case err != nil: - dlog.Errorf(ctx, "Failed to retrieve agent config for workload %s.%s: %v", wl.GetName(), wl.GetNamespace(), err) - return nil, err case scx == nil: - if ia == "enabled" { - // A race condition may occur when a workload with "enabled" is applied. - // The workload event handler will create the agent config, but the webhook injection call may arrive before - // that agent config has been created. When this happens, we must force a second event when the creation - // is complete. - a.agentConfigs.registerPrematureInjectEvent(wl) - dlog.Debugf(ctx, "No agent config has been generated for annotation enabled %s.%s", wl.GetName(), wl.GetNamespace()) - } else { - dlog.Debugf(ctx, "Skipping %s.%s (no agent config)", wl.GetName(), wl.GetNamespace()) - } + dlog.Debugf(ctx, "Skipping %s.%s (no agent config)", wl.GetName(), wl.GetNamespace()) return nil, nil case scx.AgentConfig().Manual: dlog.Debugf(ctx, "Skipping webhook where agent is manually injected %s.%s", wl.GetName(), wl.GetNamespace()) @@ -238,7 +226,7 @@ func (a *agentInjector) Uninstall(ctx context.Context) { func needInitContainer(config *agentconfig.Sidecar) bool { for _, cc := range config.Containers { - if !cc.Replace { + if cc.Replace == agentconfig.ReplacePolicyIntercept { for _, ic := range cc.Intercepts { if ic.Headless || ic.TargetPortNumeric { return true @@ -254,7 +242,7 @@ func maybeRemoveAppContainer(pod *core.Pod, config *agentconfig.Sidecar, patches cns := pod.Spec.Containers for i := len(cns) - 1; i >= 0; i-- { for _, cc := range config.Containers { - if cc.Name == cns[i].Name && cc.Replace { + if cc.Name == cns[i].Name && cc.Replace == agentconfig.ReplacePolicyContainer { patches = append(patches, PatchOperation{ Op: "remove", Path: fmt.Sprintf("/spec/containers/%d", i), @@ -488,7 +476,7 @@ func addPullSecrets( // addTPEnv adds telepresence specific environment variables to all interceptable app containers. func addTPEnv(pod *core.Pod, config *agentconfig.Sidecar, env map[string]string, patches PatchOps) PatchOps { agentconfig.EachContainer(pod, config, func(app *core.Container, cc *agentconfig.Container) { - if !cc.Replace { + if cc.Replace != agentconfig.ReplacePolicyContainer { patches = addContainerTPEnv(pod, app, env, patches) } }) @@ -547,13 +535,13 @@ func addContainerTPEnv(pod *core.Pod, cn *core.Container, env map[string]string, // the same replacement on all references to that port from the probes of the container. func hidePorts(pod *core.Pod, config *agentconfig.Sidecar, patches PatchOps) PatchOps { agentconfig.EachContainer(pod, config, func(app *core.Container, cc *agentconfig.Container) { - if !cc.Replace { + if cc.Replace == agentconfig.ReplacePolicyIntercept { for _, ic := range agentconfig.PortUniqueIntercepts(cc) { if ic.Headless || ic.TargetPortNumeric { // Rely on iptables mapping instead of port renames continue } - patches = hideContainerPorts(pod, app, bool(cc.Replace), ic.ContainerPortName, patches) + patches = hideContainerPorts(pod, app, ic.ContainerPortName, patches) } } }) @@ -562,7 +550,7 @@ func hidePorts(pod *core.Pod, config *agentconfig.Sidecar, patches PatchOps) Pat // hideContainerPorts will replace the symbolic name of a container port with a generated name. It will perform // the same replacement on all references to that port from the probes of the container. -func hideContainerPorts(pod *core.Pod, app *core.Container, isReplace bool, portName string, patches PatchOps) PatchOps { +func hideContainerPorts(pod *core.Pod, app *core.Container, portName string, patches PatchOps) PatchOps { cns := pod.Spec.Containers var containerPath string for i := range cns { @@ -590,20 +578,18 @@ func hideContainerPorts(pod *core.Pod, app *core.Container, isReplace bool, port // A replacing intercept will swap the app-container for one that doesn't have any // probes, so the patch must not contain renames for those. - if !isReplace { - probes := []*core.Probe{app.LivenessProbe, app.ReadinessProbe, app.StartupProbe} - probeNames := []string{"livenessProbe/", "readinessProbe/", "startupProbe/"} + probes := []*core.Probe{app.LivenessProbe, app.ReadinessProbe, app.StartupProbe} + probeNames := []string{"livenessProbe/", "readinessProbe/", "startupProbe/"} - for i, probe := range probes { - if probe == nil { - continue - } - if h := probe.HTTPGet; h != nil && h.Port.StrVal == portName { - hidePort(probeNames[i] + "httpGet/port") - } - if t := probe.TCPSocket; t != nil && t.Port.StrVal == portName { - hidePort(probeNames[i] + "tcpSocket/port") - } + for i, probe := range probes { + if probe == nil { + continue + } + if h := probe.HTTPGet; h != nil && h.Port.StrVal == portName { + hidePort(probeNames[i] + "httpGet/port") + } + if t := probe.TCPSocket; t != nil && t.Port.StrVal == portName { + hidePort(probeNames[i] + "tcpSocket/port") } } return patches diff --git a/cmd/traffic/cmd/manager/mutator/agent_injector_test.go b/cmd/traffic/cmd/manager/mutator/agent_injector_test.go index 63cb9018cf..490c93f5a2 100644 --- a/cmd/traffic/cmd/manager/mutator/agent_injector_test.go +++ b/cmd/traffic/cmd/manager/mutator/agent_injector_test.go @@ -955,8 +955,9 @@ matchExpressions: } } - podObjectMetaInjected := func(name string) meta.ObjectMeta { + podObjectMetaInjected := func(name string, sidecar *agentconfig.Sidecar) meta.ObjectMeta { pm := podObjectMeta(name) + pm.Annotations[agentconfig.ConfigAnnotation] = marshalConfig(t, sidecar) pm.Labels[agentconfig.WorkloadNameLabel] = name pm.Labels[agentconfig.WorkloadKindLabel] = "Deployment" pm.Labels[agentconfig.WorkloadEnabledLabel] = "true" @@ -1162,6 +1163,11 @@ matchExpressions: env: - name: _TEL_APP_A_SOME_NAME value: some value + - name: AGENT_CONFIG + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['telepresence.getambassador.io/agent-config'] - name: _TEL_AGENT_POD_IP valueFrom: fieldRef: @@ -1187,8 +1193,6 @@ matchExpressions: volumeMounts: - mountPath: /tel_pod_info name: traffic-annotations - - mountPath: /etc/traffic-agent - name: traffic-config - mountPath: /tel_app_exports name: export-volume - mountPath: /tmp @@ -1203,12 +1207,6 @@ matchExpressions: fieldPath: metadata.annotations path: annotations name: traffic-annotations - - configMap: - items: - - key: named-port - path: config.yaml - name: telepresence-agents - name: traffic-config - emptyDir: {} name: export-volume - emptyDir: {} @@ -1216,6 +1214,11 @@ matchExpressions: - op: replace path: /spec/containers/0/ports/0/name value: tm-http +- op: replace + path: /metadata/annotations + value: + telepresence.getambassador.io/agent-config: '%s' + telepresence.getambassador.io/inject-traffic-agent: enabled - op: replace path: /metadata/labels value: @@ -1254,6 +1257,11 @@ matchExpressions: env: - name: TELEPRESENCE_API_PORT value: "9981" + - name: AGENT_CONFIG + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['telepresence.getambassador.io/agent-config'] - name: _TEL_AGENT_POD_IP valueFrom: fieldRef: @@ -1279,8 +1287,6 @@ matchExpressions: volumeMounts: - mountPath: /tel_pod_info name: traffic-annotations - - mountPath: /etc/traffic-agent - name: traffic-config - mountPath: /tel_app_exports name: export-volume - mountPath: /tmp @@ -1295,12 +1301,6 @@ matchExpressions: fieldPath: metadata.annotations path: annotations name: traffic-annotations - - configMap: - items: - - key: named-port - path: config.yaml - name: telepresence-agents - name: traffic-config - emptyDir: {} name: export-volume - emptyDir: {} @@ -1308,6 +1308,11 @@ matchExpressions: - op: replace path: /spec/containers/0/ports/0/name value: tm-http +- op: replace + path: /metadata/annotations + value: + telepresence.getambassador.io/agent-config: '%s' + telepresence.getambassador.io/inject-traffic-agent: enabled - op: replace path: /metadata/labels value: @@ -1395,6 +1400,11 @@ matchExpressions: args: - agent env: + - name: AGENT_CONFIG + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['telepresence.getambassador.io/agent-config'] - name: _TEL_AGENT_POD_IP valueFrom: fieldRef: @@ -1420,8 +1430,6 @@ matchExpressions: volumeMounts: - mountPath: /tel_pod_info name: traffic-annotations - - mountPath: /etc/traffic-agent - name: traffic-config - mountPath: /tel_app_exports name: export-volume - mountPath: /tmp @@ -1436,12 +1444,6 @@ matchExpressions: fieldPath: metadata.annotations path: annotations name: traffic-annotations - - configMap: - items: - - key: named-port - path: config.yaml - name: telepresence-agents - name: traffic-config - emptyDir: {} name: export-volume - emptyDir: {} @@ -1449,6 +1451,12 @@ matchExpressions: - op: replace path: /spec/containers/0/ports/0/name value: tm-http +- op: replace + path: /metadata/annotations + value: + telepresence.getambassador.io/agent-config: '%s' + telepresence.getambassador.io/inject-service-name: named-port + telepresence.getambassador.io/inject-traffic-agent: enabled - op: replace path: /metadata/labels value: @@ -1482,6 +1490,11 @@ matchExpressions: - agent-init env: - name: LOG_LEVEL + - name: AGENT_CONFIG + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['telepresence.getambassador.io/agent-config'] - name: POD_IP valueFrom: fieldRef: @@ -1494,15 +1507,17 @@ matchExpressions: capabilities: add: - NET_ADMIN - volumeMounts: - - mountPath: /etc/traffic-agent - name: traffic-config - op: add path: /spec/containers/- value: args: - agent env: + - name: AGENT_CONFIG + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['telepresence.getambassador.io/agent-config'] - name: _TEL_AGENT_POD_IP valueFrom: fieldRef: @@ -1527,8 +1542,6 @@ matchExpressions: volumeMounts: - mountPath: /tel_pod_info name: traffic-annotations - - mountPath: /etc/traffic-agent - name: traffic-config - mountPath: /tel_app_exports name: export-volume - mountPath: /tmp @@ -1543,16 +1556,15 @@ matchExpressions: fieldPath: metadata.annotations path: annotations name: traffic-annotations - - configMap: - items: - - key: numeric-port - path: config.yaml - name: telepresence-agents - name: traffic-config - emptyDir: {} name: export-volume - emptyDir: {} name: tel-agent-tmp +- op: replace + path: /metadata/annotations + value: + telepresence.getambassador.io/agent-config: '%s' + telepresence.getambassador.io/inject-traffic-agent: enabled - op: replace path: /metadata/labels value: @@ -1590,6 +1602,11 @@ matchExpressions: - agent-init env: - name: LOG_LEVEL + - name: AGENT_CONFIG + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['telepresence.getambassador.io/agent-config'] - name: POD_IP valueFrom: fieldRef: @@ -1602,15 +1619,17 @@ matchExpressions: capabilities: add: - NET_ADMIN - volumeMounts: - - mountPath: /etc/traffic-agent - name: traffic-config - op: add path: /spec/containers/- value: args: - agent env: + - name: AGENT_CONFIG + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['telepresence.getambassador.io/agent-config'] - name: _TEL_AGENT_POD_IP valueFrom: fieldRef: @@ -1635,8 +1654,6 @@ matchExpressions: volumeMounts: - mountPath: /tel_pod_info name: traffic-annotations - - mountPath: /etc/traffic-agent - name: traffic-config - mountPath: /tel_app_exports name: export-volume - mountPath: /tmp @@ -1651,16 +1668,15 @@ matchExpressions: fieldPath: metadata.annotations path: annotations name: traffic-annotations - - configMap: - items: - - key: numeric-port - path: config.yaml - name: telepresence-agents - name: traffic-config - emptyDir: {} name: export-volume - emptyDir: {} name: tel-agent-tmp +- op: replace + path: /metadata/annotations + value: + telepresence.getambassador.io/agent-config: '%s' + telepresence.getambassador.io/inject-traffic-agent: enabled - op: replace path: /metadata/labels value: @@ -1675,16 +1691,63 @@ matchExpressions: { "Apply Patch: re-processing, null patch", &core.Pod{ - ObjectMeta: podObjectMetaInjected("numeric-port"), + ObjectMeta: podObjectMetaInjected("numeric-port", &agentconfig.Sidecar{ + AgentImage: "ghcr.io/telepresenceio/tel2:2.13.3", + AgentName: "numeric-port", + Namespace: "some-ns", + WorkloadName: "numeric-port", + WorkloadKind: "Deployment", + ManagerHost: "traffic-manager.default", + ManagerPort: 8081, + APIPort: 0, + Containers: []*agentconfig.Container{ + { + Name: "some-container", + Intercepts: []*agentconfig.Intercept{ + { + ServiceName: "numeric-port", + TargetPortNumeric: true, + Protocol: "TCP", + ContainerPort: 8888, + ServicePort: 80, + AgentPort: 9900, + }, + }, + EnvPrefix: "A_", + MountPoint: "/tel_app_mounts/some-container", + Replace: agentconfig.ReplacePolicyIntercept, + }, + }, + SecurityContext: nil, + }), Spec: core.PodSpec{ InitContainers: []core.Container{{ Name: agentconfig.InitContainerName, Image: "ghcr.io/telepresenceio/tel2:2.13.3", Args: []string{"agent-init"}, - VolumeMounts: []core.VolumeMount{{ - Name: agentconfig.ConfigVolumeName, - MountPath: agentconfig.ConfigMountPoint, - }}, + Env: []core.EnvVar{ + { + Name: "LOG_LEVEL", + }, + { + Name: "AGENT_CONFIG", + ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.annotations['telepresence.getambassador.io/agent-config']", + }, + }, + }, + { + Name: "POD_IP", + ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "status.podIP", + }, + }, + }, + }, SecurityContext: &core.SecurityContext{ Capabilities: &core.Capabilities{ Add: []core.Capability{"NET_ADMIN"}, @@ -1708,6 +1771,15 @@ matchExpressions: }}, EnvFrom: nil, Env: []core.EnvVar{ + { + Name: "AGENT_CONFIG", + ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.annotations['telepresence.getambassador.io/agent-config']", + }, + }, + }, { Name: "_TEL_AGENT_POD_IP", ValueFrom: &core.EnvVarSource{ @@ -1735,10 +1807,6 @@ matchExpressions: Name: "traffic-annotations", MountPath: "/tel_pod_info", }, - { - Name: "traffic-config", - MountPath: "/etc/traffic-agent", - }, { Name: "export-volume", MountPath: "/tel_app_exports", @@ -1824,6 +1892,11 @@ matchExpressions: value: default-secret-name - name: _TEL_APP_A_BOTH_NAMES value: $(_TEL_APP_A_TOKEN_VOLUME) and $(_TEL_APP_A_SECRET_NAME) + - name: AGENT_CONFIG + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations['telepresence.getambassador.io/agent-config'] - name: _TEL_AGENT_POD_IP valueFrom: fieldRef: @@ -1854,8 +1927,6 @@ matchExpressions: readOnly: true - mountPath: /tel_pod_info name: traffic-annotations - - mountPath: /etc/traffic-agent - name: traffic-config - mountPath: /tel_app_exports name: export-volume - mountPath: /tmp @@ -1870,15 +1941,6 @@ matchExpressions: fieldPath: metadata.annotations path: annotations name: traffic-annotations -- op: add - path: /spec/volumes/- - value: - configMap: - items: - - key: named-port - path: config.yaml - name: telepresence-agents - name: traffic-config - op: add path: /spec/volumes/- value: @@ -1892,6 +1954,11 @@ matchExpressions: - op: replace path: /spec/containers/0/ports/0/name value: tm-http +- op: replace + path: /metadata/annotations + value: + telepresence.getambassador.io/agent-config: '%s' + telepresence.getambassador.io/inject-traffic-agent: enabled - op: replace path: /metadata/labels value: @@ -1940,12 +2007,14 @@ matchExpressions: cw := GetMap(ctx) var actualPatch PatchOps var actualErr error + var cfgJSON string if test.generateConfig { gc, err := agentmap.GeneratorConfigFunc("ghcr.io/telepresenceio/tel2:2.13.3") require.NoError(t, err) var scx agentconfig.SidecarExt if scx, actualErr = generateForPod(t, ctx, test.pod, gc); actualErr == nil { - actualErr = cw.store(ctx, scx) + cw.Store(scx) + cfgJSON = marshalConfig(t, scx) } } if actualErr == nil { @@ -1955,20 +2024,27 @@ matchExpressions: } requireContains(t, actualErr, strings.ReplaceAll(test.expectedError, "", test.pod.Name)) if actualPatch != nil || test.expectedPatch != "" { + expectedPatch := test.expectedPatch + if expectedPatch != "null\n" { + expectedPatch = fmt.Sprintf(expectedPatch, cfgJSON) + } patchBytes, err := json.Marshal(actualPatch, json.Deterministic(true), jsonv1.OmitEmptyWithLegacyDefinition(true), json.FormatNilSliceAsNull(true)) require.NoError(t, err) patchBytes, err = yaml.JSONToYAML(patchBytes) require.NoError(t, err) patchString := string(patchBytes) - if test.expectedPatch != patchString { - fmt.Println(patchString) - } - assert.Equal(t, test.expectedPatch, patchString, "patches differ") + assert.Equal(t, expectedPatch, patchString, "patches differ") } }) } } +func marshalConfig(t *testing.T, sce agentconfig.SidecarExt) string { + cfgJSON, err := agentconfig.MarshalTight(sce) + require.NoError(t, err) + return cfgJSON +} + func requireContains(t *testing.T, err error, expected string) { if expected == "" { require.NoError(t, err) @@ -2047,7 +2123,6 @@ func setupAgentInjector(t *testing.T, ctx context.Context, ci kubernetes.Interfa cw := NewWatcher() ctx = WithMap(ctx, cw) - cw.DisableRollouts() cw.Start(ctx) require.NoError(t, cw.StartWatchers(ctx)) diff --git a/cmd/traffic/cmd/manager/mutator/configmap_watcher.go b/cmd/traffic/cmd/manager/mutator/configmap_watcher.go deleted file mode 100644 index 24d70f76d8..0000000000 --- a/cmd/traffic/cmd/manager/mutator/configmap_watcher.go +++ /dev/null @@ -1,131 +0,0 @@ -package mutator - -import ( - "context" - "fmt" - - core "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - informerCore "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/tools/cache" - - "github.com/datawire/dlib/dlog" - "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" - "github.com/telepresenceio/telepresence/v2/pkg/informer" -) - -func tpAgentsInformer(ctx context.Context, ns string) informerCore.ConfigMapInformer { - f := informer.GetK8sFactory(ctx, ns) - cV1 := informerCore.New(f, ns, func(options *meta.ListOptions) { - options.FieldSelector = "metadata.name=" + agentconfig.ConfigMap - }) - cms := cV1.ConfigMaps() - return cms -} - -func tpAgentsConfigMap(ctx context.Context, ns string) (*core.ConfigMap, error) { - cm, err := tpAgentsInformer(ctx, ns).Lister().ConfigMaps(ns).Get(agentconfig.ConfigMap) - if err != nil { - if !errors.IsNotFound(err) { - return nil, fmt.Errorf("unable to get ConfigMap %s: %w", agentconfig.ConfigMap, err) - } - cm = nil - } - return cm, nil -} - -func (c *configWatcher) startConfigMap(ctx context.Context, ns string) cache.SharedIndexInformer { - ix := tpAgentsInformer(ctx, ns).Informer() - _ = ix.SetTransform(func(o any) (any, error) { - // Strip of the parts of the service that we don't care about - if cm, ok := o.(*core.ConfigMap); ok { - cm.ManagedFields = nil - cm.Finalizers = nil - cm.OwnerReferences = nil - } - return o, nil - }) - _ = ix.SetWatchErrorHandler(func(_ *cache.Reflector, err error) { - dlog.Errorf(ctx, "watcher for ConfigMap %s %s: %v", agentconfig.ConfigMap, whereWeWatch(ns), err) - }) - return ix -} - -func (c *configWatcher) watchConfigMap(ctx context.Context, ix cache.SharedIndexInformer) (cache.ResourceEventHandlerRegistration, error) { - return ix.AddEventHandler( - cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj any) { - if cm, ok := obj.(*core.ConfigMap); ok { - dlog.Debugf(ctx, "ADDED %s.%s", cm.Name, cm.Namespace) - c.getNamespaceLock(cm.Namespace) - c.handleAdd(ctx, cm) - } - }, - DeleteFunc: func(obj any) { - cm, ok := obj.(*core.ConfigMap) - if !ok { - if dfu, isDfu := obj.(*cache.DeletedFinalStateUnknown); isDfu { - cm, ok = dfu.Obj.(*core.ConfigMap) - } - } - if ok { - dlog.Debugf(ctx, "DELETED %s.%s", cm.Name, cm.Namespace) - c.getNamespaceLock(cm.Namespace) - c.handleDelete(ctx, cm) - } - }, - UpdateFunc: func(oldObj, newObj any) { - if cm, ok := newObj.(*core.ConfigMap); ok { - dlog.Debugf(ctx, "UPDATED %s.%s", cm.Name, cm.Namespace) - c.handleUpdate(ctx, oldObj.(*core.ConfigMap), cm) - } - }, - }) -} - -func (c *configWatcher) handleAdd(ctx context.Context, cm *core.ConfigMap) { - ns := cm.Namespace - for n, yml := range cm.Data { - c.handleAddOrUpdateEntry(ctx, entry{ - name: n, - namespace: ns, - value: yml, - }) - } -} - -func (c *configWatcher) handleDelete(ctx context.Context, cm *core.ConfigMap) { - ns := cm.Namespace - for n, yml := range cm.Data { - c.handleDeleteEntry(ctx, entry{ - name: n, - namespace: ns, - value: yml, - }) - } -} - -func (c *configWatcher) handleUpdate(ctx context.Context, oldCm, newCm *core.ConfigMap) { - ns := newCm.Namespace - for n, newYml := range newCm.Data { - e := entry{ - name: n, - namespace: ns, - value: newYml, - } - if oldYml, ok := oldCm.Data[n]; ok { - e.oldValue = oldYml - } - c.handleAddOrUpdateEntry(ctx, e) - } - for n, oldYml := range oldCm.Data { - if _, ok := newCm.Data[n]; !ok { - c.handleDeleteEntry(ctx, entry{ - name: n, - namespace: ns, - value: oldYml, - }) - } - } -} diff --git a/cmd/traffic/cmd/manager/mutator/constants.go b/cmd/traffic/cmd/manager/mutator/constants.go index 7ba9fbf1ff..60068e3289 100644 --- a/cmd/traffic/cmd/manager/mutator/constants.go +++ b/cmd/traffic/cmd/manager/mutator/constants.go @@ -2,11 +2,9 @@ package mutator import ( "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" - "github.com/telepresenceio/telepresence/v2/pkg/workload" ) const ( - InjectAnnotation = workload.DomainPrefix + "inject-" + agentconfig.ContainerName - ServiceNameAnnotation = workload.DomainPrefix + "inject-service-name" - ManualInjectAnnotation = workload.DomainPrefix + "manually-injected" + InjectAnnotation = agentconfig.DomainPrefix + "inject-" + agentconfig.ContainerName + ServiceNameAnnotation = agentconfig.DomainPrefix + "inject-service-name" ) diff --git a/cmd/traffic/cmd/manager/mutator/service_watcher.go b/cmd/traffic/cmd/manager/mutator/service_watcher.go index 5f9f180cbe..309ddcb52b 100644 --- a/cmd/traffic/cmd/manager/mutator/service_watcher.go +++ b/cmd/traffic/cmd/manager/mutator/service_watcher.go @@ -23,7 +23,7 @@ type affectedConfig struct { scx agentconfig.SidecarExt } -func (c *configWatcher) configsAffectedBySvc(ctx context.Context, nsData map[string]string, svc *core.Service, trustUID bool) []affectedConfig { +func (c *configWatcher) configsAffectedBySvc(ctx context.Context, svc *core.Service, trustUID bool) []affectedConfig { references := func(ac *agentconfig.Sidecar) (k8sapi.Workload, error, bool) { for _, cn := range ac.Containers { for _, ic := range cn.Intercepts { @@ -46,27 +46,21 @@ func (c *configWatcher) configsAffectedBySvc(ctx context.Context, nsData map[str } var affected []affectedConfig - for _, cfg := range nsData { - scx, err := agentconfig.UnmarshalYAML([]byte(cfg)) - if err != nil { - dlog.Errorf(ctx, "failed to decode ConfigMap entry %q into an agent config", cfg) - } else if wl, err, ok := references(scx.AgentConfig()); ok { - affected = append(affected, affectedConfig{scx: scx, wl: wl, err: err}) + c.agentConfigs.Compute(svc.Namespace, func(sceMap map[string]agentconfig.SidecarExt, loaded bool) (map[string]agentconfig.SidecarExt, bool) { + if loaded { + for _, scx := range sceMap { + if wl, err, ok := references(scx.AgentConfig()); ok { + affected = append(affected, affectedConfig{scx: scx, wl: wl, err: err}) + } + } } - } + return sceMap, !loaded + }) return affected } func (c *configWatcher) affectedConfigs(ctx context.Context, svc *core.Service, trustUID bool) []affectedConfig { - ns := svc.Namespace - nsData, err := data(ctx, ns) - if err != nil { - return nil - } - if len(nsData) == 0 { - return nil - } - return c.configsAffectedBySvc(ctx, nsData, svc, trustUID) + return c.configsAffectedBySvc(ctx, svc, trustUID) } func (c *configWatcher) startServices(ctx context.Context, ns string) cache.SharedIndexInformer { @@ -136,9 +130,7 @@ func (c *configWatcher) updateSvc(ctx context.Context, svc *core.Service, trustU if err != nil { if errors.IsNotFound(err) { dlog.Debugf(ctx, "Deleting config entry for %s %s.%s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace) - if err = c.remove(ctx, ac.AgentName, ac.Namespace); err != nil { - dlog.Error(ctx, err) - } + c.Delete(ac.AgentName, ac.Namespace) } else { dlog.Error(ctx, err) } @@ -149,15 +141,17 @@ func (c *configWatcher) updateSvc(ctx context.Context, svc *core.Service, trustU acn, err := cfg.Generate(ctx, wl, ac) if err != nil { if strings.Contains(err.Error(), "unable to find") { - if err = c.remove(ctx, ac.AgentName, ac.Namespace); err != nil { - dlog.Error(ctx, err) - } + c.Delete(ac.AgentName, ac.Namespace) } else { dlog.Error(ctx, err) } continue } - if err = c.store(ctx, acn); err != nil { + ac = acn.AgentConfig() + c.Store(acn) + dlog.Debugf(ctx, "deleting pods with config mismatch for %s %s.%s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace) + err = c.DeletePodsWithConfigMismatch(ctx, acn) + if err != nil { dlog.Error(ctx, err) } } diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index 924ee2da47..c374e383ac 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "slices" - "strings" "sync" "sync/atomic" "time" @@ -13,49 +12,41 @@ import ( "github.com/puzpuzpuz/xsync/v3" "google.golang.org/protobuf/types/known/durationpb" core "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/api/policy/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/retry" "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dlog" - "github.com/datawire/dlib/dtime" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/managerutil" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/namespaces" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/informer" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" - "github.com/telepresenceio/telepresence/v2/pkg/maps" "github.com/telepresenceio/telepresence/v2/pkg/workload" ) type Map interface { - Get(context.Context, string, string) (agentconfig.SidecarExt, error) + Get(string, string) agentconfig.SidecarExt + Store(agentconfig.SidecarExt) Start(context.Context) StartWatchers(context.Context) error Wait(context.Context) error OnAdd(context.Context, k8sapi.Workload, agentconfig.SidecarExt) error OnDelete(context.Context, string, string) error DeleteMapsAndRolloutAll(context.Context) - Blacklist(podName, namespace string) - Whitelist(podName, namespace string) - IsBlacklisted(podName, namespace string) bool - DisableRollouts() - - store(ctx context.Context, acx agentconfig.SidecarExt) error - remove(ctx context.Context, name, namespace string) error + IsInactive(podName string) bool + Inactivate(podName string) + DeletePodsWithConfig(ctx context.Context, wl k8sapi.Workload) error + DeletePodsWithConfigMismatch(ctx context.Context, scx agentconfig.SidecarExt) error + DeleteAllPodsWithConfig(ctx context.Context, namespace string) error RegenerateAgentMaps(ctx context.Context, s string) error - Delete(ctx context.Context, name, namespace string) error - Update(ctx context.Context, namespace string, updater func(cm *core.ConfigMap) (bool, error)) error - - registerPrematureInjectEvent(wl k8sapi.Workload) + Delete(name, namespace string) + Update(name, namespace string, updater func(cm agentconfig.SidecarExt) (agentconfig.SidecarExt, error)) (agentconfig.SidecarExt, error) } var NewWatcherFunc = NewWatcher //nolint:gochecknoglobals // extension point @@ -79,264 +70,6 @@ func Load(ctx context.Context) Map { return cw } -func (e *entry) workload(ctx context.Context) (agentconfig.SidecarExt, k8sapi.Workload, error) { - scx, err := agentconfig.UnmarshalYAML([]byte(e.value)) - if err != nil { - return nil, nil, fmt.Errorf("failed to decode ConfigMap entry %q into an agent config", e.value) - } - ac := scx.AgentConfig() - wl, err := agentmap.GetWorkload(ctx, ac.WorkloadName, ac.Namespace, ac.WorkloadKind) - if err != nil { - return nil, nil, err - } - return scx, wl, nil -} - -// isRolloutNeeded checks if the agent's entry in telepresence-agents matches the actual state of the -// pods. If it does, then there's no reason to trigger a rollout. -func (c *configWatcher) isRolloutNeeded(ctx context.Context, wl k8sapi.Workload, ac *agentconfig.Sidecar) bool { - if c.rolloutDisabled { - return false - } - podMeta := wl.GetPodTemplate().GetObjectMeta() - if wl.GetDeletionTimestamp() != nil { - return false - } - - injectAnnotation, ok := podMeta.GetAnnotations()[agentconfig.InjectAnnotation] - if ok { - // Annotation controls injection, so no explicit rollout is needed unless the deployment was added before the - // traffic-manager or the traffic-manager already received an injection event but failed due to the lack - // of an agent config. - if c.running.Load() { - if c.receivedPrematureInjectEvent(wl) { - dlog.Debugf(ctx, "Rollout of %s.%s is necessary. Pod template has inject annotation %s and a premature injection event was received", - wl.GetName(), wl.GetNamespace(), injectAnnotation) - return true - } - } - } - podLabels := podMeta.GetLabels() - if len(podLabels) == 0 { - // Have never seen this, but if it happens, then rollout only if an agent is desired - dlog.Debugf(ctx, "Rollout of %s.%s is necessary. Pod template has no pod labels", - wl.GetName(), wl.GetNamespace()) - return true - } - - selector := labels.SelectorFromValidatedSet(podLabels) - podsAPI := informer.GetK8sFactory(ctx, wl.GetNamespace()).Core().V1().Pods().Lister().Pods(wl.GetNamespace()) - pods, err := podsAPI.List(selector) - if err != nil { - dlog.Debugf(ctx, "Rollout of %s.%s is necessary. Unable to retrieve current pods: %v", - wl.GetName(), wl.GetNamespace(), err) - return true - } - - runningPods := 0 - okPods := 0 - var rolloutReasons []string - for _, pod := range pods { - if c.IsBlacklisted(pod.Name, pod.Namespace) { - dlog.Debugf(ctx, "Skipping blacklisted pod %s.%s", pod.Name, pod.Namespace) - continue - } - if !agentmap.IsPodRunning(pod) { - continue - } - runningPods++ - if ror := isRolloutNeededForPod(ctx, ac, wl.GetName(), wl.GetNamespace(), pod); ror != "" { - if !slices.Contains(rolloutReasons, ror) { - rolloutReasons = append(rolloutReasons, ror) - } - } else { - okPods++ - } - } - // Rollout if there are no running pods - if runningPods == 0 { - if injectAnnotation != "" && wl.GetCreationTimestamp().After(c.startedAt) { - dlog.Debugf(ctx, "Rollout of %s.%s is not necessary. Pod template has inject annotation %s", - wl.GetName(), wl.GetNamespace(), injectAnnotation) - return false - } - if ac != nil { - dlog.Debugf(ctx, "Rollout of %s.%s is necessary. An agent is desired and there are no pods", - wl.GetName(), wl.GetNamespace()) - return true - } - return false - } - if okPods == 0 { - // Found no pods out there that match the desired state - for _, ror := range rolloutReasons { - dlog.Debug(ctx, ror) - } - return true - } - if ac == nil { - if okPods < runningPods { - dlog.Debugf(ctx, "Rollout of %s.%s is necessary. At least one pod still has an agent", - wl.GetName(), wl.GetNamespace()) - return true - } - return false - } - dlog.Debugf(ctx, "Rollout of %s.%s is not necessary. At least one pod have the desired agent state", - wl.GetName(), wl.GetNamespace()) - return false -} - -func isRolloutNeededForPod(ctx context.Context, ac *agentconfig.Sidecar, name, namespace string, pod *core.Pod) string { - podAc := agentmap.AgentContainer(pod) - if ac == nil { - if podAc == nil { - dlog.Debugf(ctx, "Rollout check for %s.%s is found that no agent is desired and no agent config is present for pod %s", name, namespace, pod.GetName()) - return "" - } - return fmt.Sprintf("Rollout of %s.%s is necessary. No agent is desired but the pod %s has one", name, namespace, pod.GetName()) - } - if podAc == nil { - // Rollout because an agent is desired but the pod doesn't have one - return fmt.Sprintf("Rollout of %s.%s is necessary. An agent is desired but the pod %s doesn't have one", - name, namespace, pod.GetName()) - } - desiredAc, anns := agentconfig.AgentContainer(ctx, pod, ac) - if !(containerEqual(ctx, podAc, desiredAc) && maps.Equal(anns, pod.ObjectMeta.Annotations)) { - return fmt.Sprintf("Rollout of %s.%s is necessary. The desired agent is not equal to the existing agent in pod %s", - name, namespace, pod.GetName()) - } - podIc := agentmap.InitContainer(pod) - if podIc == nil { - if needInitContainer(ac) { - return fmt.Sprintf("Rollout of %s.%s is necessary. An init-container is desired but the pod %s doesn't have one", - name, namespace, pod.GetName()) - } - } else { - if !needInitContainer(ac) { - return fmt.Sprintf("Rollout of %s.%s is necessary. No init-container is desired but the pod %s has one", - name, namespace, pod.GetName()) - } - } - for _, cn := range ac.Containers { - var found *core.Container - cns := pod.Spec.Containers - for i := range cns { - if cns[i].Name == cn.Name { - found = &cns[i] - break - } - } - if cn.Replace { - if found != nil { - return fmt.Sprintf("Rollout of %s.%s is necessary. The %s container must be replaced", - name, namespace, cn.Name) - } - } else if found == nil { - return fmt.Sprintf("Rollout of %s.%s is necessary. The %s container must be restored", - name, namespace, cn.Name) - } - } - return "" -} - -func (c *configWatcher) triggerRollout(ctx context.Context, wl k8sapi.Workload, ac *agentconfig.Sidecar) { - lck := c.getRolloutLock(wl) - if !lck.TryLock() { - // A rollout is already in progress, doing it again once it is complete wouldn't do any good. - return - } - defer lck.Unlock() - - if !c.isRolloutNeeded(ctx, wl, ac) { - return - } - - switch wl.GetKind() { - case "StatefulSet", "ReplicaSet": - triggerScalingRollout(ctx, wl) - default: - restartAnnotation := generateRestartAnnotationPatch(wl.GetPodTemplate()) - if err := wl.Patch(ctx, types.JSONPatchType, []byte(restartAnnotation)); err != nil { - err = fmt.Errorf("unable to patch %s %s.%s: %v", wl.GetKind(), wl.GetName(), wl.GetNamespace(), err) - dlog.Error(ctx, err) - return - } - dlog.Infof(ctx, "Successfully rolled out %s.%s", wl.GetName(), wl.GetNamespace()) - } -} - -// generateRestartAnnotationPatch generates a JSON patch that adds or updates the annotation -// We need to use this particular patch type because argo-rollouts do not support strategic merge patches. -func generateRestartAnnotationPatch(podTemplate *core.PodTemplateSpec) string { - basePointer := "/spec/template/metadata/annotations" - pointer := fmt.Sprintf( - basePointer+"/%s", - strings.ReplaceAll(workload.AnnRestartedAt, "/", "~1"), - ) - - if _, ok := podTemplate.Annotations[workload.AnnRestartedAt]; ok { - return fmt.Sprintf( - `[{"op": "replace", "path": "%s", "value": "%s"}]`, pointer, time.Now().Format(time.RFC3339), - ) - } - - if len(podTemplate.Annotations) == 0 { - return fmt.Sprintf( - `[{"op": "add", "path": "%s", "value": {}}, {"op": "add", "path": "%s", "value": "%s"}]`, basePointer, pointer, time.Now().Format(time.RFC3339), - ) - } - - return fmt.Sprintf( - `[{"op": "add", "path": "%s", "value": "%s"}]`, pointer, time.Now().Format(time.RFC3339), - ) -} - -func triggerScalingRollout(ctx context.Context, wl k8sapi.Workload) { - // Rollout of a replicatset/statefulset will not recreate the pods. In order for that to happen, the - // set must be scaled down to zero replicas and then up again. - dlog.Debugf(ctx, "Performing %s rollout of %s.%s using scaling", wl.GetKind(), wl.GetName(), wl.GetNamespace()) - replicas := wl.Replicas() - if replicas == 0 { - dlog.Debugf(ctx, "%s %s.%s has zero replicas so rollout was a no-op", wl.GetKind(), wl.GetName(), wl.GetNamespace()) - return - } - - waitForReplicaCount := func(count int) error { - for retryCount := 0; retryCount < 200; retryCount++ { - if nwl, err := k8sapi.GetWorkload(ctx, wl.GetName(), wl.GetNamespace(), wl.GetKind()); err == nil { - if rp := nwl.Replicas(); rp == count { - wl = nwl - return nil - } - } - dtime.SleepWithContext(ctx, 300*time.Millisecond) - } - return fmt.Errorf("%s %s.%s never scaled down to zero", wl.GetKind(), wl.GetName(), wl.GetNamespace()) - } - - patch := `{"spec": {"replicas": 0}}` - if err := wl.Patch(ctx, types.StrategicMergePatchType, []byte(patch)); err != nil { - err = fmt.Errorf("unable to scale %s %s.%s to zero: %w", wl.GetKind(), wl.GetName(), wl.GetNamespace(), err) - dlog.Error(ctx, err) - return - } - if err := waitForReplicaCount(0); err != nil { - dlog.Error(ctx, err) - return - } - dlog.Debugf(ctx, "%s %s.%s was scaled down to zero. Scaling back to %d", wl.GetKind(), wl.GetName(), wl.GetNamespace(), replicas) - patch = fmt.Sprintf(`{"spec": {"replicas": %d}}`, replicas) - if err := wl.Patch(ctx, types.StrategicMergePatchType, []byte(patch)); err != nil { - err = fmt.Errorf("unable to scale %s %s.%s to %d: %v", wl.GetKind(), wl.GetName(), wl.GetNamespace(), replicas, err) - dlog.Error(ctx, err) - } - if err := waitForReplicaCount(replicas); err != nil { - dlog.Error(ctx, err) - return - } -} - // RegenerateAgentMaps load the telepresence-agents config map, regenerates all entries in it, // and then, if any of the entries changed, it updates the map. func (c *configWatcher) RegenerateAgentMaps(ctx context.Context, agentImage string) error { @@ -357,63 +90,54 @@ func (c *configWatcher) RegenerateAgentMaps(ctx context.Context, agentImage stri // and then, if any of the entries changed, it updates the map. func (c *configWatcher) regenerateAgentMaps(ctx context.Context, ns string, gc agentmap.GeneratorConfig) error { dlog.Debugf(ctx, "regenerate agent maps %s", whereWeWatch(ns)) - lister := tpAgentsInformer(ctx, ns).Lister() - cml, err := lister.List(labels.Everything()) + pods, err := podList(ctx, "", "", ns) if err != nil { return err } + dbpCmp := cmp.Comparer(func(a, b *durationpb.Duration) bool { return a.AsDuration() == b.AsDuration() }) - n := len(cml) - for i := 0; i < n; i++ { - cm := cml[i] - changed := false - ns := cm.Namespace - err = c.Update(ctx, ns, func(cm *core.ConfigMap) (bool, error) { - dlog.Debugf(ctx, "regenerate: checking namespace %s", ns) - data := cm.Data - for n, d := range data { - e := &entry{name: n, namespace: ns, value: d} - acx, wl, err := e.workload(ctx) - if err != nil { - if !errors.IsNotFound(err) { - return false, err - } - dlog.Debugf(ctx, "regenereate: no workload found %s", n) - delete(data, n) // Workload no longer exists - changed = true - continue - } - ncx, err := gc.Generate(ctx, wl, acx) - if err != nil { - return false, err - } - if cmp.Equal(acx, ncx, dbpCmp) { - dlog.Debugf(ctx, "regenereate: agent %s is not modified", n) - continue - } - yml, err := ncx.Marshal() - if err != nil { - return false, err - } - dlog.Debugf(ctx, "regenereate: agent %s was regenerated", n) - data[n] = string(yml) - changed = true + wls := make(map[workloadKey]agentconfig.SidecarExt, len(pods)) + for _, pod := range pods { + cfgJSON, ok := pod.ObjectMeta.Annotations[agentconfig.ConfigAnnotation] + if !ok { + continue + } + sce, err := agentconfig.UnmarshalJSON(cfgJSON) + if err != nil { + dlog.Errorf(ctx, "unable to unmarshal agent config from annotation in pod %s.%s: %v", pod.Name, pod.Namespace, err) + continue + } + ac := sce.AgentConfig() + key := workloadKey{ + name: ac.WorkloadName, + namespace: ac.Namespace, + kind: ac.WorkloadKind, + } + newSce, ok := wls[key] + if !ok { + wl, err := agentmap.GetWorkload(ctx, ac.WorkloadName, ac.Namespace, ac.WorkloadKind) + if err != nil { + dlog.Errorf(ctx, "unable to load %s %s.%s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace) + continue } - if changed { - dlog.Debugf(ctx, "regenereate: updating regenerated agents") + newSce, err = gc.Generate(ctx, wl, sce) + if err != nil { + dlog.Errorf(ctx, "unable to update config for %s %s.%s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace) + continue } - return changed, nil - }) + wls[key] = newSce + c.Store(newSce) + } + if !cmp.Equal(newSce, sce, dbpCmp) { + go func() { + deletePod(ctx, pod) + }() + } } - return err -} - -type podKey struct { - name string - namespace string + return nil } type workloadKey struct { @@ -423,8 +147,7 @@ type workloadKey struct { } const ( - configMapWatcher = iota - serviceWatcher + serviceWatcher = iota deploymentWatcher replicaSetWatcher statefulSetWatcher @@ -438,131 +161,87 @@ type informersWithCancel struct { eventRegs [watcherMax]cache.ResourceEventHandlerRegistration } -func newWorkloadKey(wl k8sapi.Workload) workloadKey { - return workloadKey{ - name: wl.GetName(), - namespace: wl.GetNamespace(), - kind: wl.GetKind(), - } +type inactivation struct { + time.Time + deleted bool } type configWatcher struct { - cancel context.CancelFunc - rolloutLocks *xsync.MapOf[workloadKey, *sync.Mutex] - nsLocks *xsync.MapOf[string, *sync.RWMutex] - blacklistedPods *xsync.MapOf[podKey, time.Time] - prematureInjectionEvents *xsync.MapOf[workloadKey, time.Time] - informers *xsync.MapOf[string, *informersWithCancel] - startedAt time.Time - rolloutDisabled bool - running atomic.Bool + cancel context.CancelFunc + agentConfigs *xsync.MapOf[string, map[string]agentconfig.SidecarExt] + nsLocks *xsync.MapOf[string, *sync.RWMutex] + informers *xsync.MapOf[string, *informersWithCancel] + inactivePods *xsync.MapOf[string, inactivation] + startedAt time.Time + terminating atomic.Bool self Map // For extension } -// Blacklist will prevent the pod from being used when determining if a rollout is necessary, and -// from participating in ReviewIntercept calls. This is needed because there's a lag between the -// time when a pod is deleted and its agent announces its departure during which the pod must be -// considered inactive. -func (c *configWatcher) Blacklist(podName, namespace string) { - c.blacklistedPods.Store(podKey{name: podName, namespace: namespace}, time.Now()) -} - -func (c *configWatcher) Whitelist(podName, namespace string) { - c.blacklistedPods.Delete(podKey{name: podName, namespace: namespace}) -} - -func (c *configWatcher) registerPrematureInjectEvent(wl k8sapi.Workload) { - c.prematureInjectionEvents.Store(newWorkloadKey(wl), time.Now()) -} - -func (c *configWatcher) receivedPrematureInjectEvent(wl k8sapi.Workload) bool { - _, yes := c.prematureInjectionEvents.Load(newWorkloadKey(wl)) - return yes -} - -func (c *configWatcher) DisableRollouts() { - c.rolloutDisabled = true -} - -func (c *configWatcher) IsBlacklisted(podName, namespace string) bool { - _, yes := c.blacklistedPods.Load(podKey{name: podName, namespace: namespace}) - return yes -} - -func (c *configWatcher) Delete(ctx context.Context, name, namespace string) error { - return c.remove(ctx, name, namespace) +func (c *configWatcher) Delete(name, namespace string) { + c.agentConfigs.Compute(namespace, func(sceMap map[string]agentconfig.SidecarExt, loaded bool) (map[string]agentconfig.SidecarExt, bool) { + if loaded { + delete(sceMap, name) + return sceMap, len(sceMap) == 0 + } + return nil, true + }) } -func (c *configWatcher) Update(ctx context.Context, namespace string, updater func(cm *core.ConfigMap) (bool, error)) error { - api := k8sapi.GetK8sInterface(ctx).CoreV1().ConfigMaps(namespace) - return retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - defer func() { - if r := recover(); r != nil { - err = derror.PanicToError(r) - dlog.Errorf(ctx, "%+v", err) +func (c *configWatcher) Update(name, namespace string, updater func(agentconfig.SidecarExt) (agentconfig.SidecarExt, error)) (agentconfig.SidecarExt, error) { + var err error + var sce agentconfig.SidecarExt + c.agentConfigs.Compute(namespace, func(sceMap map[string]agentconfig.SidecarExt, loaded bool) (map[string]agentconfig.SidecarExt, bool) { + if loaded { + var ok bool + sce, ok = sceMap[name] + if ok { + sce = sce.Clone() } - }() - lock := c.getNamespaceLock(namespace) - lock.Lock() - defer lock.Unlock() - cm, err := tpAgentsConfigMap(ctx, namespace) - if err != nil { - return err - } - cm = cm.DeepCopy() // Protect the cached cm from updates - create := cm == nil - if create { - cm = &core.ConfigMap{ - TypeMeta: meta.TypeMeta{ - Kind: "ConfigMap", - APIVersion: "v1", - }, - ObjectMeta: meta.ObjectMeta{ - Name: agentconfig.ConfigMap, - Namespace: namespace, - }, + sce, err = updater(sce) + if err == nil { + if sce == nil { + delete(sceMap, name) + } else { + sceMap[name] = sce + } + } + return sceMap, false + } else { + sce, err = updater(nil) + if err == nil && sce != nil { + sceMap = map[string]agentconfig.SidecarExt{name: sce} + return sceMap, false } + return nil, true } + }) + return sce, err +} - changed, err := updater(cm) - if err == nil && changed { - if create { - _, err = api.Create(ctx, cm, meta.CreateOptions{}) - if err != nil && errors.IsAlreadyExists(err) { - // Treat AlreadyExists as a Conflict so that this attempt is retried. - err = errors.NewConflict(schema.GroupResource{ - Group: "v1", - Resource: "ConfigMap", - }, cm.Name, err) - } - } else { - _, err = api.Update(ctx, cm, meta.UpdateOptions{}) - } +func (c *configWatcher) Store(sce agentconfig.SidecarExt) { + ag := sce.AgentConfig() + c.agentConfigs.Compute(ag.Namespace, func(sceMap map[string]agentconfig.SidecarExt, loaded bool) (map[string]agentconfig.SidecarExt, bool) { + if loaded { + sceMap[ag.AgentName] = sce + } else { + sceMap = map[string]agentconfig.SidecarExt{ag.AgentName: sce} } - return err + return sceMap, false }) } func NewWatcher() Map { w := &configWatcher{ - nsLocks: xsync.NewMapOf[string, *sync.RWMutex](), - rolloutLocks: xsync.NewMapOf[workloadKey, *sync.Mutex](), - informers: xsync.NewMapOf[string, *informersWithCancel](), - blacklistedPods: xsync.NewMapOf[podKey, time.Time](), - prematureInjectionEvents: xsync.NewMapOf[workloadKey, time.Time](), + nsLocks: xsync.NewMapOf[string, *sync.RWMutex](), + informers: xsync.NewMapOf[string, *informersWithCancel](), + inactivePods: xsync.NewMapOf[string, inactivation](), + agentConfigs: xsync.NewMapOf[string, map[string]agentconfig.SidecarExt](), } w.self = w return w } -type entry struct { - name string - namespace string - value string - oldValue string -} - func (c *configWatcher) SetSelf(self Map) { c.self = self } @@ -577,7 +256,6 @@ func (c *configWatcher) startInformers(ctx context.Context, ns string) (iwc *inf }() ifns := [watcherMax]cache.SharedIndexInformer{} - ifns[configMapWatcher] = c.startConfigMap(ctx, ns) ifns[serviceWatcher] = c.startServices(ctx, ns) for _, wlKind := range managerutil.GetEnv(ctx).EnabledWorkloadKinds { switch wlKind { @@ -609,10 +287,6 @@ func (c *configWatcher) startInformers(ctx context.Context, ns string) (iwc *inf func (c *configWatcher) startWatchers(ctx context.Context, iwc *informersWithCancel) (err error) { ifns := iwc.informers - iwc.eventRegs[configMapWatcher], err = c.watchConfigMap(ctx, ifns[configMapWatcher]) - if err != nil { - return err - } iwc.eventRegs[serviceWatcher], err = c.watchServices(ctx, ifns[serviceWatcher]) if err != nil { return err @@ -657,7 +331,6 @@ func (c *configWatcher) Wait(ctx context.Context) error { } func (c *configWatcher) OnAdd(ctx context.Context, wl k8sapi.Workload, acx agentconfig.SidecarExt) error { - c.triggerRollout(ctx, wl, acx.AgentConfig()) return nil } @@ -665,55 +338,6 @@ func (c *configWatcher) OnDelete(context.Context, string, string) error { return nil } -func (c *configWatcher) handleAddOrUpdateEntry(ctx context.Context, e entry) { - switch e.oldValue { - case e.value: - return - case "": - dlog.Debugf(ctx, "add %s.%s", e.name, e.namespace) - default: - dlog.Debugf(ctx, "update %s.%s", e.name, e.namespace) - } - scx, wl, err := e.workload(ctx) - if err != nil { - if !errors.IsNotFound(err) { - dlog.Error(ctx, err) - } - return - } - ac := scx.AgentConfig() - if ac.Manual { - // Manually added, just ignore - return - } - if err = c.self.OnAdd(ctx, wl, scx); err != nil { - dlog.Error(ctx, err) - } -} - -func (c *configWatcher) handleDeleteEntry(ctx context.Context, e entry) { - dlog.Debugf(ctx, "del %s.%s", e.name, e.namespace) - scx, wl, err := e.workload(ctx) - if err != nil { - if !errors.IsNotFound(err) { - dlog.Error(ctx, err) - return - } - } else { - ac := scx.AgentConfig() - if ac.Create || ac.Manual { - // Deleted before it was generated or manually added, just ignore - return - } - } - if err = c.self.OnDelete(ctx, e.name, e.namespace); err != nil { - dlog.Error(ctx, err) - } - if wl != nil { - c.triggerRollout(ctx, wl, nil) - } -} - func (c *configWatcher) getNamespaceLock(ns string) *sync.RWMutex { lock, _ := c.nsLocks.LoadOrCompute(ns, func() *sync.RWMutex { return &sync.RWMutex{} @@ -721,92 +345,18 @@ func (c *configWatcher) getNamespaceLock(ns string) *sync.RWMutex { return lock } -func (c *configWatcher) getRolloutLock(wl k8sapi.Workload) *sync.Mutex { - lock, _ := c.rolloutLocks.LoadOrCompute(newWorkloadKey(wl), func() *sync.Mutex { - return &sync.Mutex{} - }) - return lock -} - // Get returns the Sidecar configuration that for the given key and namespace. // If no configuration is found, this function returns nil, nil. // An error is only returned when the configmap holding the configuration could not be loaded for // other reasons than it did not exist. -func (c *configWatcher) Get(ctx context.Context, key, ns string) (agentconfig.SidecarExt, error) { - lock := c.getNamespaceLock(ns) - lock.RLock() - defer lock.RUnlock() - - data, err := data(ctx, ns) - if err != nil { - return nil, err - } - v, ok := data[key] - if !ok { - return nil, nil - } - return agentconfig.UnmarshalYAML([]byte(v)) -} - -// remove will delete an agent config from the agents ConfigMap for the given namespace. It will -// also update the current snapshot. -// An attempt to delete a manually added config is a no-op. -func (c *configWatcher) remove(ctx context.Context, name, namespace string) error { - return c.Update(ctx, namespace, func(cm *core.ConfigMap) (bool, error) { - yml, ok := cm.Data[name] - if !ok { - return false, nil - } - scx, err := agentconfig.UnmarshalYAML([]byte(yml)) - if err != nil { - return false, err - } - if scx.AgentConfig().Manual { - return false, nil - } - delete(cm.Data, name) - dlog.Debugf(ctx, "Deleting %s from ConfigMap %s.%s", name, agentconfig.ConfigMap, namespace) - return true, nil - }) -} - -// store an agent config in the agents ConfigMap for the given namespace. -func (c *configWatcher) store(ctx context.Context, acx agentconfig.SidecarExt) error { - js, err := acx.Marshal() - yml := string(js) - if err != nil { - return err - } - ac := acx.AgentConfig() - ns := ac.Namespace - return c.Update(ctx, ns, func(cm *core.ConfigMap) (bool, error) { - if cm.Data == nil { - cm.Data = make(map[string]string) - } else { - if oldYml, ok := cm.Data[ac.AgentName]; ok { - if oldYml == yml { - return false, nil - } - dlog.Debugf(ctx, "Modifying configmap entry for sidecar %s.%s", ac.AgentName, ac.Namespace) - scx, err := agentconfig.UnmarshalYAML([]byte(oldYml)) - if err == nil && scx.AgentConfig().Manual { - dlog.Warnf(ctx, "avoided an attempt to overwrite manually added Config entry for %s.%s", ac.AgentName, ns) - return false, nil - } - } +func (c *configWatcher) Get(key, ns string) (ac agentconfig.SidecarExt) { + c.agentConfigs.Compute(ns, func(sceMap map[string]agentconfig.SidecarExt, loaded bool) (map[string]agentconfig.SidecarExt, bool) { + if loaded { + ac = sceMap[key] } - cm.Data[ac.AgentName] = yml - dlog.Debugf(ctx, "updating agent %s in %s.%s", ac.AgentName, agentconfig.ConfigMap, ns) - return true, nil + return sceMap, !loaded }) -} - -func data(ctx context.Context, ns string) (map[string]string, error) { - cm, err := tpAgentsConfigMap(ctx, ns) - if err != nil || cm == nil { - return nil, err - } - return cm.Data, nil + return ac } func whereWeWatch(ns string) string { @@ -822,7 +372,6 @@ func (c *configWatcher) startPods(ctx context.Context, ns string) cache.SharedIn _ = ix.SetTransform(func(o any) (any, error) { if pod, ok := o.(*core.Pod); ok { pod.ManagedFields = nil - pod.OwnerReferences = nil pod.Finalizers = nil ps := &pod.Status @@ -855,18 +404,10 @@ func (c *configWatcher) startPods(ctx context.Context, ns string) cache.SharedIn return ix } -func (c *configWatcher) gcBlacklisted(now time.Time) { - const maxAge = time.Minute - maxCreated := now.Add(-maxAge) - c.blacklistedPods.Range(func(key podKey, created time.Time) bool { - if created.Before(maxCreated) { - c.blacklistedPods.Delete(key) - } - return true - }) - c.prematureInjectionEvents.Range(func(key workloadKey, created time.Time) bool { - if created.Before(maxCreated) { - c.prematureInjectionEvents.Delete(key) +func (c *configWatcher) gcInactivated(now time.Time) { + c.inactivePods.Range(func(key string, value inactivation) bool { + if now.Sub(value.Time) > time.Minute { + c.inactivePods.Delete(key) } return true }) @@ -881,7 +422,7 @@ func (c *configWatcher) Start(ctx context.Context) { ticker.Stop() return case now := <-ticker.C: - c.gcBlacklisted(now) + c.gcInactivated(now) } } }() @@ -960,7 +501,6 @@ func (c *configWatcher) deleteMapsAndRolloutNS(ctx context.Context, ns string, i lock := c.getNamespaceLock(ns) lock.Lock() defer func() { - lock.Unlock() c.nsLocks.Delete(ns) c.informers.Delete(ns) }() @@ -972,33 +512,172 @@ func (c *configWatcher) deleteMapsAndRolloutNS(ctx context.Context, ns string, i } } iwc.cancel() + lock.Unlock() + + err := c.DeleteAllPodsWithConfig(ctx, ns) + if err != nil { + dlog.Errorf(ctx, "unable to delete agents in namespace %s: %v", ns, err) + } +} - now := meta.NewDeleteOptions(0) - api := k8sapi.GetK8sInterface(ctx).CoreV1() - wlm, err := data(ctx, ns) +func (c *configWatcher) DeletePodsWithConfigMismatch(ctx context.Context, scx agentconfig.SidecarExt) error { + ac := scx.AgentConfig() + pods, err := podList(ctx, ac.WorkloadKind, ac.AgentName, ac.Namespace) if err != nil { - dlog.Errorf(ctx, "unable to get configmap %s.%s: %v", agentconfig.ConfigMap, ns, err) - return + return err + } + cfgJSON, err := agentconfig.MarshalTight(scx) + if err != nil { + return err } - for k, v := range wlm { - e := &entry{name: k, namespace: ns, value: v} - scx, wl, err := e.workload(ctx) + + for _, pod := range pods { + err = c.DeleteIfMismatch(ctx, pod, cfgJSON) if err != nil { - if !errors.IsNotFound(err) { - dlog.Errorf(ctx, "unable to get workload for %s.%s %s: %v", k, ns, v, err) - } - continue + return err } - ac := scx.AgentConfig() - if ac.Create || ac.Manual { - // Deleted before it was generated or manually added, just ignore - continue + } + return nil +} + +func (c *configWatcher) DeletePodsWithConfig(ctx context.Context, wl k8sapi.Workload) error { + pods, err := podList(ctx, wl.GetKind(), wl.GetName(), wl.GetNamespace()) + if err != nil { + return err + } + + for _, pod := range pods { + err = c.DeleteIfMismatch(ctx, pod, "") + if err != nil { + return err + } + } + return nil +} + +func (c *configWatcher) DeleteAllPodsWithConfig(ctx context.Context, namespace string) error { + c.agentConfigs.Delete(namespace) + pods, err := podList(ctx, "", "", namespace) + if err != nil { + return err + } + for _, pod := range pods { + err = c.DeleteIfMismatch(ctx, pod, "") + if err != nil { + return err } - c.triggerRollout(ctx, wl, nil) } - if err := api.ConfigMaps(ns).Delete(ctx, agentconfig.ConfigMap, *now); err != nil { - if !errors.IsNotFound(err) { - dlog.Errorf(ctx, "unable to delete ConfigMap %s-%s: %v", agentconfig.ConfigMap, ns, err) + return nil +} + +func (c *configWatcher) DeleteIfMismatch(ctx context.Context, pod *core.Pod, cfgJSON string) error { + if c.IsDeleted(pod.Name) { + dlog.Debugf(ctx, "Skipping pod %s because it is already deleted", pod.Name) + return nil + } + a := pod.ObjectMeta.Annotations + if v, ok := a[agentconfig.ManualInjectAnnotation]; ok && v == "true" { + dlog.Debugf(ctx, "Skipping pod %s because it is managed manually", pod.Name) + return nil + } + if a[agentconfig.ConfigAnnotation] == cfgJSON { + dlog.Debugf(ctx, "Keeping pod %s because its config is still valid", pod.Name) + return nil + } + var err error + c.inactivePods.Compute(pod.Name, func(v inactivation, loaded bool) (inactivation, bool) { + if loaded && v.deleted { + dlog.Debugf(ctx, "Skipping pod %s because it was deleted by another thread", pod.Name) + return v, false + } + dlog.Debugf(ctx, "Deleting pod %s because its config is no longer valid", pod.Name) + go func() { + deletePod(ctx, pod) + }() + return inactivation{Time: time.Now(), deleted: true}, false + }) + return err +} + +func deletePod(ctx context.Context, pod *core.Pod) { + dlog.Debugf(ctx, "Evicting pod %s", pod.Name) + err := k8sapi.GetK8sInterface(ctx).CoreV1().Pods(pod.Namespace).EvictV1(ctx, &v1.Eviction{ + ObjectMeta: meta.ObjectMeta{Name: pod.Name, Namespace: pod.Namespace}, + }) + if err != nil { + dlog.Errorf(ctx, "Failed to evict pod %s: %v", pod.Name, err) + } +} + +func (c *configWatcher) Inactivate(podName string) { + c.inactivePods.LoadOrCompute(podName, func() inactivation { + return inactivation{Time: time.Now()} + }) +} + +func (c *configWatcher) IsDeleted(podName string) bool { + v, ok := c.inactivePods.Load(podName) + return ok && v.deleted +} + +func (c *configWatcher) IsInactive(podName string) bool { + _, ok := c.inactivePods.Load(podName) + return ok +} + +func podIsRunning(pod *core.Pod) bool { + switch pod.Status.Phase { + case core.PodPending, core.PodRunning: + return true + default: + return false + } +} + +func podList(ctx context.Context, kind, name, namespace string) ([]*core.Pod, error) { + var lister interface { + List(selector labels.Selector) (ret []*core.Pod, err error) + } + api := informer.GetK8sFactory(ctx, namespace).Core().V1().Pods().Lister() + if namespace == "" { + lister = api + } else { + lister = api.Pods(namespace) + } + pods, err := lister.List(labels.Everything()) + if err != nil { + return nil, fmt.Errorf("error listing pods in namespace %s: %v", namespace, err) + } + enabledWorkloads := managerutil.GetEnv(ctx).EnabledWorkloadKinds + supportedKinds := make([]string, len(enabledWorkloads)) + for i, wlKind := range enabledWorkloads { + switch wlKind { + case workload.DeploymentKind: + supportedKinds[i] = "Deployment" + case workload.ReplicaSetKind: + supportedKinds[i] = "ReplicaSet" + case workload.StatefulSetKind: + supportedKinds[i] = "StatefulSet" + case workload.RolloutKind: + supportedKinds[i] = "Rollout" + } + } + var podsOfInterest []*core.Pod + for _, pod := range pods { + if !podIsRunning(pod) { + continue + } + dlog.Debugf(ctx, "getting owner workload for pod %s.%s", pod.Name, pod.Namespace) + wl, err := agentmap.FindOwnerWorkload(ctx, k8sapi.Pod(pod), supportedKinds) + if err == nil { + dlog.Debugf(ctx, "owner workload for pod %s.%s is %s %s", pod.Name, pod.Namespace, wl.GetKind(), wl.GetName()) + if (kind == "" || wl.GetKind() == kind) && (name == "" || wl.GetName() == name) { + dlog.Debugf(ctx, "owner workload was a match. Pod %s.%s is of interest", pod.Name, pod.Namespace) + podsOfInterest = append(podsOfInterest, pod) + } + } else { + dlog.Debugf(ctx, "error getting owner workload for pod %s.%s: %v", pod.Name, pod.Namespace, err) } } + return podsOfInterest, nil } diff --git a/cmd/traffic/cmd/manager/mutator/workload_watcher.go b/cmd/traffic/cmd/manager/mutator/workload_watcher.go index e3586bd4cd..ab16ea74be 100644 --- a/cmd/traffic/cmd/manager/mutator/workload_watcher.go +++ b/cmd/traffic/cmd/manager/mutator/workload_watcher.go @@ -28,11 +28,11 @@ func (c *configWatcher) watchWorkloads(ctx context.Context, ix cache.SharedIndex DeleteFunc: func(obj any) { if wl, ok := workload.FromAny(obj); ok { if len(wl.GetOwnerReferences()) == 0 { - c.deleteWorkload(ctx, wl) + c.Delete(wl.GetName(), wl.GetNamespace()) } } else if dfsu, ok := obj.(*cache.DeletedFinalStateUnknown); ok { if wl, ok = workload.FromAny(dfsu.Obj); ok && len(wl.GetOwnerReferences()) == 0 { - c.deleteWorkload(ctx, wl) + c.Delete(wl.GetName(), wl.GetNamespace()) } } }, @@ -46,31 +46,19 @@ func (c *configWatcher) watchWorkloads(ctx context.Context, ix cache.SharedIndex }) } -func (c *configWatcher) deleteWorkload(ctx context.Context, wl k8sapi.Workload) { - scx, err := c.Get(ctx, wl.GetName(), wl.GetNamespace()) - if err != nil { - dlog.Errorf(ctx, "Failed to get sidecar config: %v", err) - } else if scx != nil { - err = c.Delete(ctx, wl.GetName(), wl.GetNamespace()) - if err != nil { - dlog.Errorf(ctx, "Failed to delete sidecar config: %v", err) - } - } -} - func (c *configWatcher) updateWorkload(ctx context.Context, wl, oldWl k8sapi.Workload, state workload.State) { if state == workload.StateFailure { return } tpl := wl.GetPodTemplate() - ia, ok := tpl.Annotations[workload.InjectAnnotation] + ia, ok := tpl.Annotations[agentconfig.InjectAnnotation] if !ok { return } if oldWl != nil && cmp.Equal(oldWl.GetPodTemplate(), tpl, cmpopts.IgnoreFields(meta.ObjectMeta{}, "Namespace", "UID", "ResourceVersion", "CreationTimestamp", "DeletionTimestamp"), cmpopts.IgnoreMapEntries(func(k, _ string) bool { - return k == workload.AnnRestartedAt + return k == agentconfig.RestartedAtAnnotation })) { return } @@ -88,11 +76,7 @@ func (c *configWatcher) updateWorkload(ctx context.Context, wl, oldWl k8sapi.Wor } var scx agentconfig.SidecarExt if oldWl != nil { - scx, err = c.Get(ctx, wl.GetName(), wl.GetNamespace()) - if err != nil { - dlog.Errorf(ctx, "Failed to get sidecar config: %v", err) - return - } + scx = c.Get(wl.GetName(), wl.GetNamespace()) } action := "Generating" if scx == nil { @@ -103,17 +87,25 @@ func (c *configWatcher) updateWorkload(ctx context.Context, wl, oldWl k8sapi.Wor scx, err = cfg.Generate(ctx, wl, scx) if err != nil { if strings.Contains(err.Error(), "unable to find") { - if err = c.remove(ctx, wl.GetName(), wl.GetNamespace()); err != nil { - dlog.Error(ctx, err) - } + c.Delete(wl.GetName(), wl.GetNamespace()) } else { dlog.Error(ctx, err) } + return } - if err = c.store(ctx, scx); err != nil { + + c.Store(scx) + ac := scx.AgentConfig() + dlog.Debugf(ctx, "deleting pods with config mismatch for %s %s.%s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace) + err = c.DeletePodsWithConfigMismatch(ctx, scx) + if err != nil { dlog.Error(ctx, err) } case "false", "disabled": - c.deleteWorkload(ctx, wl) + c.Delete(wl.GetName(), wl.GetNamespace()) + err := c.DeletePodsWithConfig(ctx, wl) + if err != nil { + dlog.Error(ctx, err) + } } } diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index d399d95ff9..e3f46937d4 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -210,16 +210,18 @@ func (s *service) ArriveAsClient(ctx context.Context, client *rpc.ClientInfo) (* // ArriveAsAgent establishes a session between an agent and the Manager. func (s *service) ArriveAsAgent(ctx context.Context, agent *rpc.AgentInfo) (*rpc.SessionInfo, error) { dlog.Debugf(ctx, "ArriveAsAgent %s called", agent.PodName) - if val := validateAgent(agent); val != "" { return nil, status.Error(codes.InvalidArgument, val) } - mutator.GetMap(ctx).Whitelist(agent.PodName, agent.Namespace) for _, cn := range agent.Containers { s.removeExcludedEnvVars(cn.Environment) } - sessionID := s.state.AddAgent(agent, s.clock.Now()) + + sessionID, err := s.state.AddAgent(ctx, agent, s.clock.Now()) + if err != nil { + return nil, err + } return &rpc.SessionInfo{ SessionId: sessionID, @@ -305,11 +307,10 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa case <-sessionDone: // Manager believes this session has ended. return nil - case as, ok := <-agentsCh: + case agm, ok := <-agentsCh: if !ok { return nil } - agm := as.State agents = make([]*rpc.AgentPodInfo, len(agm)) agentNames = make([]string, len(agm)) i := 0 @@ -333,7 +334,7 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa if !ok { return nil } - interceptInfos = is.State + interceptInfos = is for i, a := range agents { a.Intercepted = isIntercepted(agentNames[i], a.Namespace) } @@ -414,11 +415,11 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(string, *rp dlog.Debug(ctx, "WatchAgentsNS request cancelled") return nil } - agentSessionIDs := slices.Sorted(maps.Keys(snapshot.State)) + agentSessionIDs := slices.Sorted(maps.Keys(snapshot)) agents := make([]*rpc.AgentInfo, 0, len(agentSessionIDs)) for _, agentSessionID := range agentSessionIDs { if as := s.state.GetSession(agentSessionID); as != nil && as.Active() { - agents = append(agents, snapshot.State[agentSessionID]) + agents = append(agents, snapshot[agentSessionID]) } } if slices.EqualFunc(agents, lastSnap, infosEqual) { @@ -429,7 +430,7 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(string, *rp names := make([]string, len(agents)) i := 0 for _, a := range agents { - names[i] = a.Name + "." + a.Namespace + names[i] = a.PodName + "." + a.Namespace i++ } dlog.Debugf(ctx, "WatchAgentsNS sending update %v", names) @@ -519,8 +520,8 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W return nil } dlog.Debugf(ctx, "WatchIntercepts sending update") - intercepts := make([]*rpc.InterceptInfo, 0, len(snapshot.State)) - for _, intercept := range snapshot.State { + intercepts := make([]*rpc.InterceptInfo, 0, len(snapshot)) + for _, intercept := range snapshot { intercepts = append(intercepts, intercept) } resp := &rpc.InterceptInfoSnapshot{ @@ -612,14 +613,6 @@ func (s *service) CreateIntercept(ctx context.Context, ciReq *rpc.CreateIntercep return nil, err } - if ciReq.InterceptSpec.Replace { - err = s.state.AddInterceptFinalizer(interceptInfo.Id, s.state.RestoreAppContainer) - if err != nil { - // The intercept's been created but we can't finalize it... - dlog.Errorf(ctx, "Failed to add finalizer for %s: %v", interceptInfo.Id, err) - } - } - SetGauge(s.state.GetInterceptActiveStatus(), client.Name, client.InstallId, &spec.Name, 1) IncrementInterceptCounterFunc(s.state.GetInterceptCounter(), client.Name, client.InstallId, spec) @@ -687,7 +680,7 @@ func (s *service) ReviewIntercept(ctx context.Context, rIReq *rpc.ReviewIntercep sessionID := rIReq.GetSession().GetSessionId() ceptID := rIReq.Id - agent := s.state.GetActiveAgent(sessionID) + agent := s.state.GetAgent(sessionID) if agent == nil { return &empty.Empty{}, nil } @@ -705,7 +698,7 @@ func (s *service) ReviewIntercept(ctx context.Context, rIReq *rpc.ReviewIntercep if intercept.Spec.Namespace != agent.Namespace || intercept.Spec.Agent != agent.Name { return } - if mutator.GetMap(ctx).IsBlacklisted(agent.PodName, agent.Namespace) { + if mutator.GetMap(ctx).IsInactive(agent.PodName) { dlog.Debugf(ctx, "Pod %s.%s is blacklisted", agent.PodName, agent.Namespace) return } @@ -914,6 +907,12 @@ func (s *service) SetLogLevel(ctx context.Context, request *rpc.LogLevelRequest) return &empty.Empty{}, nil } +func (s *service) UninstallAgents(ctx context.Context, request *rpc.UninstallAgentsRequest) (*empty.Empty, error) { + ctx = managerutil.WithSessionInfo(ctx, request.GetSessionInfo()) + dlog.Debugf(ctx, "UninstallAgents called %s", request.Agents) + return &empty.Empty{}, s.state.UninstallAgents(ctx, request) +} + func (s *service) WatchLogLevel(_ *empty.Empty, stream rpc.Manager_WatchLogLevelServer) error { dlog.Debugf(stream.Context(), "WatchLogLevel called") return s.state.WaitForTempLogLevel(stream) diff --git a/cmd/traffic/cmd/manager/service_test.go b/cmd/traffic/cmd/manager/service_test.go index 61c123b731..e015df853b 100644 --- a/cmd/traffic/cmd/manager/service_test.go +++ b/cmd/traffic/cmd/manager/service_test.go @@ -186,97 +186,6 @@ func TestConnect(t *testing.T) { require.NoError(err) require.Len(aSnapA.Agents, 3) t.Logf("=> client[alice] agent snapshot = %s", dumps(aSnapA)) - - // Alice creates an intercept - - spec := &rpc.InterceptSpec{ - Name: "first", - Namespace: "default", - Client: testClients["alice"].Name, - Agent: testAgents["hello"].Name, - Mechanism: "tcp", - TargetHost: "asdf", - TargetPort: 9876, - } - - first, err := client.CreateIntercept(ctx, &rpc.CreateInterceptRequest{ - Session: aliceSess2, - InterceptSpec: spec, - }) - require.NoError(err) - require.True(proto.Equal(spec, first.Spec)) - t.Logf("=> intercept info: %s", dumps(first)) - - aSnapI, err = aliceWI.Recv() - require.NoError(err) - require.Len(aSnapI.Intercepts, 1) - require.Equal(rpc.InterceptDispositionType_WAITING, aSnapI.Intercepts[0].Disposition) - t.Logf("=> client[alice] intercept snapshot = %s", dumps(aSnapI)) - - hSnapI, err = helloWI.Recv() - require.NoError(err) - require.Len(hSnapI.Intercepts, 1) - require.Equal(rpc.InterceptDispositionType_WAITING, hSnapI.Intercepts[0].Disposition) - t.Logf("=> agent[hello] intercept snapshot = %s", dumps(hSnapI)) - - // Hello's agent reviews the intercept - - _, err = client.ReviewIntercept(ctx, &rpc.ReviewInterceptRequest{ - Session: helloSess, - Id: hSnapI.Intercepts[0].Id, - Disposition: rpc.InterceptDispositionType_ACTIVE, - Message: "okay!", - }) - require.NoError(err) - - // Causing the intercept to go active with require port assigned - - aSnapI, err = aliceWI.Recv() - require.NoError(err) - require.Len(aSnapI.Intercepts, 1) - require.Equal(rpc.InterceptDispositionType_ACTIVE, aSnapI.Intercepts[0].Disposition) - t.Logf("=> client[alice] intercept snapshot = %s", dumps(aSnapI)) - - hSnapI, err = helloWI.Recv() - require.NoError(err) - require.Len(hSnapI.Intercepts, 1) - require.Equal(rpc.InterceptDispositionType_ACTIVE, hSnapI.Intercepts[0].Disposition) - t.Logf("=> agent[hello] intercept snapshot = %s", dumps(hSnapI)) - - // Creating require duplicate intercept yields an error - - second, err := client.CreateIntercept(ctx, &rpc.CreateInterceptRequest{ - Session: aliceSess2, - InterceptSpec: spec, - }) - require.Error(err) - require.Nil(second) - t.Logf("=> intercept info: %s", dumps(second)) - - // Alice removes the intercept - - _, err = client.RemoveIntercept(ctx, &rpc.RemoveInterceptRequest2{ - Session: aliceSess2, - Name: spec.Name, - }) - require.NoError(err) - t.Logf("removed intercept") - - aSnapI, err = aliceWI.Recv() - require.NoError(err) - require.Len(aSnapI.Intercepts, 0) - t.Logf("=> client[alice] intercept snapshot = %s", dumps(aSnapI)) - - hSnapI, err = helloWI.Recv() - require.NoError(err) - require.Len(hSnapI.Intercepts, 0) - t.Logf("=> agent[hello] intercept snapshot = %s", dumps(hSnapI)) - - _, err = client.RemoveIntercept(ctx, &rpc.RemoveInterceptRequest2{ - Session: aliceSess1, // no longer require valid session, right? - Name: spec.Name, // doesn't matter... - }) - require.Error(err) _, err = client.Depart(ctx, aliceSess2) require.NoError(err) _, err = client.Depart(ctx, helloSess) @@ -306,7 +215,7 @@ func getTestClientConn(ctx context.Context, t *testing.T) *grpc.ClientConn { }, }) fakeClient.Discovery().(*fakeDiscovery.FakeDiscovery).FakedServerVersion = &k8sVersion.Info{ - GitVersion: "v1.17.0", + GitVersion: "v1.30.5", } const mgrNs = "ambassador" diff --git a/cmd/traffic/cmd/manager/state/intercept.go b/cmd/traffic/cmd/manager/state/intercept.go index 8c8776eb85..53dbf5a2e9 100644 --- a/cmd/traffic/cmd/manager/state/intercept.go +++ b/cmd/traffic/cmd/manager/state/intercept.go @@ -30,30 +30,9 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/errcat" - "github.com/telepresenceio/telepresence/v2/pkg/informer" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" ) -func (s *state) inactivateIfHasContainer(ctx context.Context, n, ns, cn string) { - api := informer.GetK8sFactory(ctx, ns).Core().V1().Pods().Lister().Pods(ns) - for sessionID, ai := range s.getAgentsByName(n, ns) { - pod, err := api.Get(ai.PodName) - if err != nil { - continue - } - cns := pod.Spec.Containers - for i := range cns { - if cns[i].Name == cn { - dlog.Debugf(ctx, "Inactivating pod %s %s because it contains container %q which is about to be replaced", pod.Name, pod.Status.PodIP, cn) - if as, ok := s.GetSession(sessionID).(*agentSessionState); ok { - as.active.Store(false) - } - break - } - } - } -} - // PrepareIntercept ensures that the given request can be matched against the intercept configuration of // the workload that it references. It returns a PreparedIntercept where all intercepted ports have been // qualified with a container port and if applicable, with service name and a service port name. @@ -95,22 +74,14 @@ func (s *state) PrepareIntercept( return interceptError(err) } + var rp agentconfig.ReplacePolicy if spec.Replace { - // If we already have an agent config, then any pod that matches the spec and also has the container that we want - // to replace must be blacklisted now, or that agent will interfere with the replace. - mm := mutator.GetMap(ctx) - scx, err := mm.Get(ctx, spec.Agent, spec.Namespace) - if err != nil { - return interceptError(err) - } - if scx != nil { - if cn, err := findContainer(scx.AgentConfig(), spec); err == nil { - s.inactivateIfHasContainer(ctx, spec.Name, spec.Namespace, cn.Name) - } - } + rp = agentconfig.ReplacePolicyContainer + } else { + rp = agentconfig.ReplacePolicyIntercept } - ac, _, err := s.ensureAgent(ctx, wl, s.isExtended(spec), spec) + ac, _, err := s.ensureAgent(ctx, wl, s.isExtended(spec), true, spec, rp) if err != nil { return interceptError(err) } @@ -145,16 +116,17 @@ func (s *state) PrepareIntercept( } func prepareAllContainerPorts(cn *agentconfig.Container, pi *rpc.PreparedIntercept) { - if ni := len(cn.Intercepts); ni > 0 { + pics := agentconfig.PortUniqueIntercepts(cn) + if ni := len(pics); ni > 0 { // Put first port in the intercept itself - i0 := cn.Intercepts[0] + i0 := pics[0] pi.ContainerPort = int32(i0.ContainerPort) pi.Protocol = string(i0.Protocol) if ni > 1 { // Put remaining ports in PodPorts with a 1:1 mapping to target port on client. pi.PodPorts = make([]string, ni-1) for i := 1; i < ni; i++ { - ic := cn.Intercepts[i] + ic := pics[i] pi.PodPorts[i-1] = fmt.Sprintf("%d:%d/%s", ic.ContainerPort, ic.ContainerPort, ic.Protocol) } } @@ -214,7 +186,7 @@ func (s *state) preparePorts(ac *agentconfig.Sidecar, cn *agentconfig.Container, } // Validate that there's no port conflict with other intercepts using the same agent. - otherIcs := s.intercepts.LoadAllMatching(func(s string, info *rpc.InterceptInfo) bool { + otherIcs := s.intercepts.LoadMatching(func(s string, info *rpc.InterceptInfo) bool { return info.Disposition == rpc.InterceptDispositionType_ACTIVE && info.Spec.Agent == ac.AgentName && info.Spec.Namespace == ac.Namespace }) @@ -248,9 +220,6 @@ func (s *state) preparePorts(ac *agentconfig.Sidecar, cn *agentconfig.Container, } func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptRequest) (*rpc.ClientInfo, *rpc.InterceptInfo, error) { - s.mu.Lock() - defer s.mu.Unlock() - clientSession := cir.Session sessionID := clientSession.SessionId client := s.GetClient(sessionID) @@ -260,6 +229,25 @@ func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptReques spec := cir.InterceptSpec interceptID := fmt.Sprintf("%s:%s", sessionID, spec.Name) + + wl, err := agentmap.GetWorkload(ctx, spec.Agent, spec.Namespace, spec.WorkloadKind) + if err != nil { + code := codes.Internal + if k8sErrors.IsNotFound(err) { + code = codes.NotFound + } + return nil, nil, status.Error(code, err.Error()) + } + + rp := agentconfig.ReplacePolicyIntercept + if spec.Replace { + rp = agentconfig.ReplacePolicyContainer + } + _, _, err = s.ensureAgent(ctx, wl, s.isExtended(spec), false, spec, rp) + if err != nil { + return nil, nil, err + } + ret, is, err := s.addIntercept(interceptID, cir) if err != nil { return nil, nil, err @@ -306,6 +294,10 @@ func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptReques return nil }) } + err = s.AddInterceptFinalizer(interceptID, s.restoreAppContainer) + if err != nil { + dlog.Errorf(ctx, "Failed to add finalizer for %s: %v", interceptID, err) + } return client, ret, nil } @@ -363,13 +355,13 @@ func (s *state) AddInterceptFinalizer(interceptID string, finalizer InterceptFin // getAgentsInterceptedByClient returns the session IDs for each agent that are currently // intercepted by the client with the given client session ID. func (s *state) getAgentsInterceptedByClient(clientSessionID string) map[string]*rpc.AgentInfo { - intercepts := s.intercepts.LoadAllMatching(func(_ string, ii *rpc.InterceptInfo) bool { + intercepts := s.intercepts.LoadMatching(func(_ string, ii *rpc.InterceptInfo) bool { return ii.ClientSession.SessionId == clientSessionID }) if len(intercepts) == 0 { return nil } - return s.agents.LoadAllMatching(func(_ string, ai *rpc.AgentInfo) bool { + return s.LoadMatchingAgents(func(_ string, ai *rpc.AgentInfo) bool { for _, ii := range intercepts { if ai.Name == ii.Spec.Agent && ai.Namespace == ii.Spec.Namespace { return true @@ -388,7 +380,7 @@ func (s *state) EnsureAgent(ctx context.Context, n, ns string) (as []*rpc.AgentI } return nil, err } - _, as, err = s.ensureAgent(ctx, wl, false, nil) + _, as, err = s.ensureAgent(ctx, wl, false, false, nil, agentconfig.ReplacePolicyInactive) return as, err } @@ -403,7 +395,7 @@ func sortAgents(as []*rpc.AgentInfo) { }) } -func (s *state) ensureAgent(parentCtx context.Context, wl k8sapi.Workload, extended bool, spec *rpc.InterceptSpec) ( +func (s *state) ensureAgent(parentCtx context.Context, wl k8sapi.Workload, extended, dryRun bool, spec *rpc.InterceptSpec, rp agentconfig.ReplacePolicy) ( ac *agentconfig.Sidecar, as []*rpc.AgentInfo, err error, ) { if agentmap.TrafficManagerSelector.Matches(labels.Set(wl.GetLabels())) { @@ -413,27 +405,37 @@ func (s *state) ensureAgent(parentCtx context.Context, wl k8sapi.Workload, exten } if !managerutil.AgentInjectorEnabled(parentCtx) { - sce, err := mutator.GetMap(parentCtx).Get(parentCtx, wl.GetName(), wl.GetNamespace()) + cfgJSON, ok := wl.GetPodTemplate().Annotations[agentconfig.ConfigAnnotation] + if !ok { + msg := fmt.Sprintf("agent-injector is disabled and no agent has been added manually for %s.%s", wl.GetName(), wl.GetNamespace()) + return nil, nil, status.Error(codes.FailedPrecondition, msg) + } + sce, err := agentconfig.UnmarshalJSON(cfgJSON) if err != nil { return nil, nil, err } - if sce != nil { - ac = sce.AgentConfig() - am := s.agents.LoadAllMatching(func(_ string, ai *rpc.AgentInfo) bool { - return ai.Name == ac.AgentName && ai.Namespace == ac.Namespace - }) - as = make([]*rpc.AgentInfo, len(am)) - i := 0 - for _, found := range am { - as[i] = found - i++ - } - sortAgents(as) - return ac, as, nil + ac = sce.AgentConfig() + am := s.LoadMatchingAgents(func(_ string, ai *rpc.AgentInfo) bool { + return ai.Name == ac.AgentName && ai.Namespace == ac.Namespace + }) + as = make([]*rpc.AgentInfo, len(am)) + i := 0 + for _, found := range am { + as[i] = found + i++ } - msg := fmt.Sprintf("agent-injector is disabled and no agent has been added manually for %s.%s", wl.GetName(), wl.GetNamespace()) - return nil, nil, status.Error(codes.FailedPrecondition, msg) + sortAgents(as) + return ac, as, nil + } + + if dryRun { + sce, err := s.getOrCreateAgentConfig(parentCtx, wl, extended, dryRun, spec, rp) + if err != nil { + return nil, nil, err + } + return sce.AgentConfig(), nil, nil } + ctx, cancel := context.WithTimeout(parentCtx, managerutil.GetEnv(parentCtx).AgentArrivalTimeout) defer cancel() @@ -442,17 +444,20 @@ func (s *state) ensureAgent(parentCtx context.Context, wl k8sapi.Workload, exten return nil, nil, err } - sce, err := s.getOrCreateAgentConfig(ctx, wl, extended, spec, false) + sce, err := s.getOrCreateAgentConfig(ctx, wl, extended, dryRun, spec, rp) + if err != nil { + return nil, nil, err + } + err = mutator.GetMap(ctx).DeletePodsWithConfigMismatch(ctx, sce) if err != nil { + dlog.Errorf(ctx, "failed to inactivate pods: %v", err) return nil, nil, err } ac = sce.AgentConfig() - if as, err = s.waitForAgents(ctx, ac.AgentName, ac.Namespace, failedCreateCh); err != nil { + if as, err = s.waitForAgents(ctx, ac, failedCreateCh); err != nil { // If no agent arrives, then drop its entry from the configmap. This ensures that there // are no false positives the next time an intercept is attempted. - if dropErr := s.dropAgentConfig(parentCtx, wl); dropErr != nil { - dlog.Errorf(ctx, "failed to remove configmap entry for %s.%s: %v", wl.GetName(), wl.GetNamespace(), dropErr) - } + s.dropAgentConfig(parentCtx, wl) return nil, nil, err } sortAgents(as) @@ -476,24 +481,19 @@ func (s *state) ValidateAgentImage(agentImage string, extended bool) (err error) func (s *state) dropAgentConfig( ctx context.Context, wl k8sapi.Workload, -) error { - return mutator.GetMap(ctx).Delete(ctx, wl.GetName(), wl.GetNamespace()) +) { + mutator.GetMap(ctx).Delete(wl.GetName(), wl.GetNamespace()) } -func (s *state) RestoreAppContainer(ctx context.Context, ii *rpc.InterceptInfo) (err error) { +func (s *state) restoreAppContainer(ctx context.Context, ii *rpc.InterceptInfo) (err error) { dlog.Debugf(ctx, "Restoring app container for %s", ii.Id) spec := ii.Spec n := spec.Agent ns := spec.Namespace mm := mutator.GetMap(ctx) - return mm.Update(ctx, ns, func(cm *core.ConfigMap) (changed bool, err error) { - y, ok := cm.Data[n] - if !ok { - return false, nil - } - sce, err := unmarshalConfigMapEntry(y, n, ns) - if err != nil { - return false, err + _, err = mm.Update(n, ns, func(sce agentconfig.SidecarExt) (ext agentconfig.SidecarExt, err error) { + if sce == nil { + return nil, nil } var cn *agentconfig.Container if spec.NoDefaultPort { @@ -501,57 +501,21 @@ func (s *state) RestoreAppContainer(ctx context.Context, ii *rpc.InterceptInfo) } else { cn, _, err = findIntercept(sce.AgentConfig(), spec) } - if !(err == nil && cn.Replace) { - return false, nil + if err != nil { + return nil, nil + } + if cn.Replace == agentconfig.ReplacePolicyInactive { + return nil, nil } - cn.Replace = false + cn.Replace = agentconfig.ReplacePolicyInactive // The pods for this workload will be killed once the new updated sidecar // reaches the configmap. We inactivate them now, so that they don't continue to // review intercepts. - for sessionID := range s.getAgentsByName(n, ns) { - if as, ok := s.GetSession(sessionID).(*agentSessionState); ok { - as.active.Store(false) - } - } - return updateSidecar(sce, cm, n) + err = mm.DeletePodsWithConfigMismatch(ctx, sce) + return sce, err }) -} - -func updateSidecar(sce agentconfig.SidecarExt, cm *core.ConfigMap, n string) (bool, error) { - yml, err := sce.Marshal() - if err != nil { - return false, err - } - oldYaml := cm.Data[n] - newYaml := string(yml) - if oldYaml != newYaml { - cm.Data[n] = newYaml - return true, nil - } - return false, nil -} - -func (s *state) waitForAgentDepartures(ctx context.Context, wl k8sapi.Workload) error { - filter := func(s string, info *rpc.AgentInfo) bool { - return info.Kind == wl.GetKind() && info.Name == wl.GetName() && info.Namespace == wl.GetNamespace() - } - if len(s.agents.LoadAllMatching(filter)) == 0 { - return nil - } - dlog.Debugf(ctx, "Waiting for deleted %s.%s agents to depart", wl.GetName(), wl.GetNamespace()) - agCh := s.agents.SubscribeSubset(ctx, filter) - for { - select { - case <-ctx.Done(): - return ctx.Err() - case as, ok := <-agCh: - if ok && len(as.State) > 0 { - continue - } - } - return nil - } + return err } func (s *state) GetOrGenerateAgentConfig(ctx context.Context, name, namespace string) (agentconfig.SidecarExt, error) { @@ -563,15 +527,31 @@ func (s *state) GetOrGenerateAgentConfig(ctx context.Context, name, namespace st } return nil, status.Error(code, err.Error()) } - return s.getOrCreateAgentConfig(ctx, wl, false, nil, true) + return s.getOrCreateAgentConfig(ctx, wl, false, true, nil, agentconfig.ReplacePolicyInactive) +} + +func (s *state) createAgentConfig(ctx context.Context, wl k8sapi.Workload, agentImage string) (sce agentconfig.SidecarExt, err error) { + var gc agentmap.GeneratorConfig + if gc, err = agentmap.GeneratorConfigFunc(agentImage); err != nil { + return nil, err + } + dlog.Debugf(ctx, "generating new agent config for %s.%s", wl.GetName(), wl.GetNamespace()) + if sce, err = gc.Generate(ctx, wl, nil); err != nil { + return nil, err + } + if err = s.self.ValidateCreateAgent(ctx, wl, sce); err != nil { + return nil, err + } + return sce, nil } func (s *state) getOrCreateAgentConfig( ctx context.Context, wl k8sapi.Workload, extended bool, - spec *rpc.InterceptSpec, dryRun bool, + spec *rpc.InterceptSpec, + rp agentconfig.ReplacePolicy, ) (sce agentconfig.SidecarExt, err error) { enabled, err := checkInterceptAnnotations(wl) if err != nil { @@ -585,40 +565,32 @@ func (s *state) getOrCreateAgentConfig( if err = s.self.ValidateAgentImage(agentImage, extended); err != nil { return nil, err } - err = mutator.GetMap(ctx).Update(ctx, wl.GetNamespace(), func(cm *core.ConfigMap) (changed bool, err error) { - doUpdate := false - y, cmFound := cm.Data[wl.GetName()] - if cmFound { - if sce, err = unmarshalConfigMapEntry(y, wl.GetName(), wl.GetNamespace()); err != nil { - return false, err - } - ac := sce.AgentConfig() + mm := mutator.GetMap(ctx) + if dryRun { + sce = mm.Get(wl.GetName(), wl.GetNamespace()) + if sce == nil { + sce, err = s.createAgentConfig(ctx, wl, agentImage) + } + return sce, err + } + + return mm.Update(wl.GetName(), wl.GetNamespace(), func(sce agentconfig.SidecarExt) (agentconfig.SidecarExt, error) { + var ac *agentconfig.Sidecar + if sce != nil { + ac = sce.AgentConfig() // If the agentImage has changed, and the extended image is requested, then update - if ac.AgentImage != agentImage && extended { + if ac.AgentImage != agentImage { ac.AgentImage = agentImage - doUpdate = true } + dlog.Debugf(ctx, "found existing agent config for %s.%s", wl.GetName(), wl.GetNamespace()) } else { - if cm.Data == nil { - cm.Data = make(map[string]string) - } - var gc agentmap.GeneratorConfig - if gc, err = agentmap.GeneratorConfigFunc(agentImage); err != nil { - return false, err - } - if sce, err = gc.Generate(ctx, wl, nil); err != nil { - return false, err - } - - // If we don't have an entry for the workload in the config-map, then all current agents for that - // workload are invalid, and we'll have to wait for them to be removed. - if err = s.waitForAgentDepartures(ctx, wl); err != nil { - return false, err + sce, err = s.createAgentConfig(ctx, wl, agentImage) + if err != nil { + return nil, err } - doUpdate = true + ac = sce.AgentConfig() } - ac := sce.AgentConfig() if spec != nil { var cn *agentconfig.Container if spec.NoDefaultPort { @@ -627,37 +599,17 @@ func (s *state) getOrCreateAgentConfig( cn, _, err = findIntercept(ac, spec) } if err != nil { - return false, err + return nil, err } - if cn.Replace != agentconfig.ReplacePolicy(spec.Replace) { - cn.Replace = agentconfig.ReplacePolicy(spec.Replace) - doUpdate = true - } - } - if dryRun { - return false, nil + cn.Replace = rp } - if doUpdate { - if cmFound { - // The pods for this workload be killed once the new updated sidecar - // reaches the configmap. We remove them now, so that they don't continue to - // review intercepts. - for sessionID := range s.getAgentsByName(wl.GetName(), wl.GetNamespace()) { - if as, ok := s.GetSession(sessionID).(*agentSessionState); ok { - as.active.Store(false) - } - } - } else { - if err = s.self.ValidateCreateAgent(ctx, wl, sce); err != nil { - return false, err - } - } - return updateSidecar(sce, cm, wl.GetName()) + if dryRun { + dlog.Debugf(ctx, "dry run for getOrCreateAgentConfig %s.%s returns", wl.GetName(), wl.GetNamespace()) + return sce, nil } - return false, nil + return sce, nil }) - return sce, err } func checkInterceptAnnotations(wl k8sapi.Workload) (bool, error) { @@ -668,7 +620,7 @@ func checkInterceptAnnotations(wl k8sapi.Workload) (bool, error) { } webhookEnabled := true - manuallyManaged := a[mutator.ManualInjectAnnotation] == "true" + manuallyManaged := a[agentconfig.ManualInjectAnnotation] == "true" ia := a[mutator.InjectAnnotation] switch ia { case "": @@ -679,7 +631,7 @@ func checkInterceptAnnotations(wl k8sapi.Workload) (bool, error) { default: return false, errcat.User.Newf( "%s is not a valid value for the %s.%s/%s annotation", - ia, wl.GetName(), wl.GetNamespace(), mutator.ManualInjectAnnotation) + ia, wl.GetName(), wl.GetNamespace(), agentconfig.ManualInjectAnnotation) } if !manuallyManaged { @@ -697,7 +649,7 @@ func checkInterceptAnnotations(wl k8sapi.Workload) (bool, error) { if an == nil { return false, errcat.User.Newf( "annotation %s.%s/%s=true but pod has no traffic-agent container", - wl.GetName(), wl.GetNamespace(), mutator.ManualInjectAnnotation) + wl.GetName(), wl.GetNamespace(), agentconfig.ManualInjectAnnotation) } return true, nil } @@ -743,7 +695,9 @@ func watchFailedInjectionEvents(ctx context.Context, name, namespace string) (<- return ec, nil } -func (s *state) waitForAgents(ctx context.Context, name, namespace string, failedCreateCh <-chan *events.Event) ([]*rpc.AgentInfo, error) { +func (s *state) waitForAgents(ctx context.Context, ac *agentconfig.Sidecar, failedCreateCh <-chan *events.Event) ([]*rpc.AgentInfo, error) { + name := ac.AgentName + namespace := ac.Namespace dlog.Debugf(ctx, "Waiting for agent %s.%s", name, namespace) snapshotCh := s.WatchAgents(ctx, func(sessionID string, agent *rpc.AgentInfo) bool { return agent.Name == name && agent.Namespace == namespace @@ -809,16 +763,17 @@ func (s *state) waitForAgents(ctx context.Context, name, namespace string, faile // The request has been canceled. return nil, status.Error(codes.Canceled, fmt.Sprintf("channel closed while waiting for agent %s.%s to arrive", name, namespace)) } - if len(snapshot.State) == 0 { + if len(snapshot) == 0 { continue } - as := make([]*rpc.AgentInfo, 0, len(snapshot.State)) - for _, a := range snapshot.State { - if mm.IsBlacklisted(a.PodName, a.Namespace) { - dlog.Debugf(ctx, "Pod %s.%s is blacklisted", a.PodName, a.Namespace) + as := make([]*rpc.AgentInfo, 0, len(snapshot)) + for _, a := range snapshot { + if mm.IsInactive(a.PodName) { + dlog.Debugf(ctx, "Agent %s.%s is blacklisted", a.PodName, a.Namespace) } else { dlog.Debugf(ctx, "Agent %s.%s is ready", a.Name, a.Namespace) as = append(as, a) + break } } if len(as) > 0 { @@ -874,14 +829,6 @@ func writeEventList(bf *strings.Builder, es []*events.Event) { } } -func unmarshalConfigMapEntry(y string, name, namespace string) (agentconfig.SidecarExt, error) { - scx, err := agentconfig.UnmarshalYAML([]byte(y)) - if err != nil { - return nil, fmt.Errorf("failed to parse entry for %s in ConfigMap %s.%s: %w", name, agentconfig.ConfigMap, namespace, err) - } - return scx, nil -} - // findContainer finds the container configuration that matches the given InterceptSpec. func findContainer(ac *agentconfig.Sidecar, spec *rpc.InterceptSpec) (foundCN *agentconfig.Container, err error) { if spec.ContainerName == "" { diff --git a/cmd/traffic/cmd/manager/state/presence_test.go b/cmd/traffic/cmd/manager/state/presence_test.go index 3f800b46ce..777b1ad0af 100644 --- a/cmd/traffic/cmd/manager/state/presence_test.go +++ b/cmd/traffic/cmd/manager/state/presence_test.go @@ -52,9 +52,10 @@ func TestPresence(t *testing.T) { a.False(isPresent("d")) collected := make([]string, 0, 3) - for id, item := range p.GetAllClients() { + p.EachClient(func(id string, item *rpc.ClientInfo) bool { collected = append(collected, fmt.Sprintf("%s/%v", id, item.Name)) - } + return true + }) a.Contains(collected, fmt.Sprintf("%s/item-a", sa)) a.Contains(collected, fmt.Sprintf("%s/item-b", sb)) a.Contains(collected, fmt.Sprintf("%s/item-c", sc)) diff --git a/cmd/traffic/cmd/manager/state/session.go b/cmd/traffic/cmd/manager/state/session.go index b3cf7a7865..28f8df3915 100644 --- a/cmd/traffic/cmd/manager/state/session.go +++ b/cmd/traffic/cmd/manager/state/session.go @@ -177,7 +177,6 @@ type agentSessionState struct { sessionState dnsRequests chan *rpc.DNSRequest dnsResponses map[string]chan *rpc.DNSResponse - active atomic.Bool } func newAgentSessionState(ctx context.Context, ts time.Time) *agentSessionState { @@ -186,16 +185,10 @@ func newAgentSessionState(ctx context.Context, ts time.Time) *agentSessionState dnsRequests: make(chan *rpc.DNSRequest), dnsResponses: make(map[string]chan *rpc.DNSResponse), } - as.active.Store(true) return as } -func (ss *agentSessionState) Active() bool { - return ss.active.Load() -} - func (ss *agentSessionState) Cancel() { - ss.active.Store(false) close(ss.dnsRequests) for k, lr := range ss.dnsResponses { delete(ss.dnsResponses, k) diff --git a/cmd/traffic/cmd/manager/state/state.go b/cmd/traffic/cmd/manager/state/state.go index 5f4d7e074e..cc1d23c680 100644 --- a/cmd/traffic/cmd/manager/state/state.go +++ b/cmd/traffic/cmd/manager/state/state.go @@ -23,6 +23,7 @@ import ( "github.com/datawire/dlib/dlog" rpc "github.com/telepresenceio/telepresence/rpc/v2/manager" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/managerutil" + "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/mutator" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/namespaces" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/watchable" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" @@ -34,7 +35,7 @@ import ( ) type State interface { - AddAgent(*rpc.AgentInfo, time.Time) string + AddAgent(context.Context, *rpc.AgentInfo, time.Time) (string, error) AddClient(*rpc.ClientInfo, time.Time) string AddIntercept(context.Context, *rpc.CreateInterceptRequest) (*rpc.ClientInfo, *rpc.InterceptInfo, error) AddInterceptFinalizer(string, InterceptFinalizer) error @@ -50,8 +51,7 @@ type State interface { ExpireSessions(context.Context, time.Time, time.Time) GetAgent(sessionID string) *rpc.AgentInfo GetOrGenerateAgentConfig(ctx context.Context, name, namespace string) (agentconfig.SidecarExt, error) - GetActiveAgent(sessionID string) *rpc.AgentInfo - GetAllClients() map[string]*rpc.ClientInfo + EachClient(f func(string, *rpc.ClientInfo) bool) GetClient(sessionID string) *rpc.ClientInfo GetSession(string) SessionState GetSessionConsumptionMetrics(string) *SessionConsumptionMetrics @@ -69,7 +69,6 @@ type State interface { PrepareIntercept(context.Context, *rpc.CreateInterceptRequest) (*rpc.PreparedIntercept, error) RemoveIntercept(context.Context, string) DropIntercept(string) - RestoreAppContainer(context.Context, *rpc.InterceptInfo) error FinalizeIntercept(ctx context.Context, intercept *rpc.InterceptInfo) LoadMatchingIntercepts(filter func(string, *rpc.InterceptInfo) bool) map[string]*rpc.InterceptInfo RemoveSession(context.Context, string) @@ -83,18 +82,18 @@ type State interface { interceptStatusGaugeVec *prometheus.GaugeVec) Tunnel(context.Context, tunnel.Stream) error UpdateIntercept(string, func(*rpc.InterceptInfo)) *rpc.InterceptInfo - UpdateClient(sessionID string, apply func(*rpc.ClientInfo)) *rpc.ClientInfo RefreshSessionConsumptionMetrics(sessionID string) ValidateAgentImage(string, bool) error WaitForTempLogLevel(rpc.Manager_WatchLogLevelServer) error - WatchAgents(context.Context, func(sessionID string, agent *rpc.AgentInfo) bool) <-chan watchable.Snapshot[*rpc.AgentInfo] + WatchAgents(context.Context, func(sessionID string, agent *rpc.AgentInfo) bool) <-chan map[string]*rpc.AgentInfo WatchDial(sessionID string) <-chan *rpc.DialRequest - WatchIntercepts(context.Context, func(sessionID string, intercept *rpc.InterceptInfo) bool) <-chan watchable.Snapshot[*rpc.InterceptInfo] + WatchIntercepts(context.Context, func(sessionID string, intercept *rpc.InterceptInfo) bool) <-chan map[string]*rpc.InterceptInfo WatchWorkloads(ctx context.Context, namespace string) (ch <-chan []workload.Event, err error) WatchLookupDNS(string) <-chan *rpc.DNSRequest ValidateCreateAgent(context.Context, k8sapi.Workload, agentconfig.SidecarExt) error NewWorkloadInfoWatcher(clientSession, namespace string) WorkloadInfoWatcher ManagesNamespace(context.Context, string) bool + UninstallAgents(context.Context, *rpc.UninstallAgentsRequest) error } type ( @@ -117,10 +116,7 @@ type state struct { // protect against memory corruption and ensure serialization for watches, we need to do our // own locking here to ensure consistency between the various maps: // - // 1. `agents` needs to stay in-sync with `sessions` - // 2. `clients` needs to stay in-sync with `sessions` // 3. `port` needs to be updated in-sync with `intercepts` - // 4. `agentsByName` needs stay in-sync with `agents` // 5. `intercepts` needs to be pruned in-sync with `clients` (based on // `intercept.ClientSession.SessionId`) // 6. `intercepts` needs to be pruned in-sync with `agents` (based on @@ -128,11 +124,10 @@ type state struct { // 7. `cfgMapLocks` access must be concurrency protected // 8. `cachedAgentImage` access must be concurrency protected // 9. `interceptState` must be concurrency protected and updated/deleted in sync with intercepts - intercepts watchable.Map[*rpc.InterceptInfo] // info for intercepts, keyed by intercept id - agents watchable.Map[*rpc.AgentInfo] // info for agent sessions, keyed by session id - clients watchable.Map[*rpc.ClientInfo] // info for client sessions, keyed by session id - sessions *xsync.MapOf[string, SessionState] // info for all sessions, keyed by session id - agentsByName *xsync.MapOf[string, *xsync.MapOf[string, *rpc.AgentInfo]] // indexed copy of `agents` + intercepts *watchable.Map[string, *rpc.InterceptInfo] // info for intercepts, keyed by intercept id + agents *watchable.Map[string, *rpc.AgentInfo] // info for agent sessions, keyed by session id + clients *xsync.MapOf[string, *rpc.ClientInfo] // info for client sessions, keyed by session id + sessions *xsync.MapOf[string, SessionState] // info for all sessions, keyed by session id interceptStates *xsync.MapOf[string, *interceptState] timedLogLevel log.TimedLevel llSubs *loglevelSubscribers @@ -155,12 +150,22 @@ func (s *state) ManagesNamespace(ctx context.Context, ns string) bool { var NewStateFunc = NewState //nolint:gochecknoglobals // extension point +func interceptEqual(a, b *rpc.InterceptInfo) bool { + return proto.Equal(a, b) +} + +func agentsEqual(a, b *rpc.AgentInfo) bool { + return proto.Equal(a, b) +} + func NewState(ctx context.Context) State { loglevel := os.Getenv("LOG_LEVEL") s := &state{ backgroundCtx: ctx, + intercepts: watchable.NewMap[string, *rpc.InterceptInfo](interceptEqual, time.Millisecond), + agents: watchable.NewMap[string, *rpc.AgentInfo](agentsEqual, time.Millisecond), + clients: xsync.NewMapOf[string, *rpc.ClientInfo](), sessions: xsync.NewMapOf[string, SessionState](), - agentsByName: xsync.NewMapOf[string, *xsync.MapOf[string, *rpc.AgentInfo]](), interceptStates: xsync.NewMapOf[string, *interceptState](), workloadWatchers: xsync.NewMapOf[string, workload.Watcher](), timedLogLevel: log.NewTimedLevel(loglevel, log.SetLevel), @@ -189,11 +194,11 @@ func NewState(ctx context.Context) State { func (s *state) pruneSessions(ctx context.Context) { nss := namespaces.Get(ctx) var sids []string - s.clients.LoadAllMatching(func(s string, c *rpc.ClientInfo) bool { + s.clients.Range(func(s string, c *rpc.ClientInfo) bool { if !slices.Contains(nss, c.Namespace) { sids = append(sids, s) } - return false + return true }) for _, sid := range sids { s.RemoveSession(ctx, sid) @@ -242,14 +247,14 @@ func (s *state) checkAgentsForIntercept(intercept *rpc.InterceptInfo) (errCode r // main //////////////////////////////////////////////////////////////// var agentList []*rpc.AgentInfo - if agentSet, ok := s.agentsByName.Load(intercept.Spec.Agent); ok { - agentSet.Range(func(_ string, agent *rpc.AgentInfo) bool { - if agent.Namespace == intercept.Spec.Namespace { - agentList = append(agentList, agent) - } - return true - }) - } + agentName := intercept.Spec.Agent + ns := intercept.Spec.Namespace + s.EachAgent(func(_ string, ai *rpc.AgentInfo) bool { + if ai.Name == agentName && ai.Namespace == ns { + agentList = append(agentList, ai) + } + return true + }) switch { case len(agentList) == 0: @@ -299,15 +304,7 @@ func (s *state) RemoveSession(ctx context.Context, sessionID string) { agent, isAgent := s.agents.LoadAndDelete(sessionID) if isAgent { - // remove it from the agentsByName index (if necessary) - dlog.Debugf(ctx, "Agent session %s. Explicit removal", agent.PodName) - s.agentsByName.Compute(agent.Name, func(ag *xsync.MapOf[string, *rpc.AgentInfo], loaded bool) (*xsync.MapOf[string, *rpc.AgentInfo], bool) { - if loaded { - ag.Delete(sessionID) - return ag, ag.Size() == 0 - } - return nil, true - }) + mutator.GetMap(s.backgroundCtx).Inactivate(agent.PodName) } else if client, isClient := s.clients.LoadAndDelete(sessionID); isClient { scm := sess.(*clientSessionState).consumptionMetrics atomic.AddUint64(&s.tunnelIngressCounter, scm.FromClientBytes.GetValue()) @@ -319,37 +316,41 @@ func (s *state) RemoveSession(ctx context.Context, sessionID string) { } func (s *state) gcSessionIntercepts(ctx context.Context, sessionID string) { - agent, isAgent := s.agents.Load(sessionID) - // GC any intercepts that relied on this session; prune any intercepts that // 1. Don't have a client session (intercept.ClientSession.SessionId) - // 2. Don't have any agents (agent.Name == intercept.Spec.Agent) - // Alternatively, if the intercept is still live but has been switched over to a different agent, send it back to WAITING state - for interceptID, intercept := range s.intercepts.LoadAll() { + // 2. Don't have any agents (agent.PodIp == intercept.PodIp) + // Alternatively, if the intercept is still live but has been switched over to a different agent, send it back to + // WAITING state + s.intercepts.Range(func(interceptID string, intercept *rpc.InterceptInfo) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED { - continue + return true } if intercept.ClientSession.SessionId == sessionID { // Client went away: // Delete it. if client := s.GetClient(sessionID); client != nil { - workload := strings.SplitN(interceptID, ":", 2)[1] - s.allInterceptsFinalizerCall(client, &workload) + wl := strings.SplitN(interceptID, ":", 2)[1] + s.allInterceptsFinalizerCall(client, &wl) } s.self.RemoveIntercept(ctx, interceptID) } else if errCode, errMsg := s.checkAgentsForIntercept(intercept); errCode != 0 { // Refcount went to zero: // Tell the client, so that the client can tell us to delete it. - intercept.Disposition = errCode - intercept.Message = errMsg - s.intercepts.Store(interceptID, intercept) - } else if isAgent && agent.PodIp == intercept.PodIp { - // The agent whose podIP was stored by the intercept is dead, but it's not the last agent - // Send it back to waiting so that one of the other agents can pick it up and set their own podIP - intercept.Disposition = rpc.InterceptDispositionType_WAITING - s.intercepts.Store(interceptID, intercept) + s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { + intercept.Disposition = errCode + intercept.Message = errMsg + }) + } else { + if agent := s.GetAgent(sessionID); agent != nil && agent.PodIp == intercept.PodIp { + // The agent whose podIP was stored by the intercept is dead, but it's not the last agent + // Send it back to waiting so that one of the other agents can pick it up and set their own podIP + s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { + intercept.Disposition = rpc.InterceptDispositionType_WAITING + }) + } } - } + return true + }) } // ExpireSessions prunes any sessions that haven't had a MarkSession heartbeat since @@ -404,20 +405,20 @@ func (s *state) GetClient(sessionID string) *rpc.ClientInfo { return ret } -func (s *state) GetAllClients() map[string]*rpc.ClientInfo { - return s.clients.LoadAll() +func (s *state) EachClient(f func(string, *rpc.ClientInfo) bool) { + s.clients.Range(f) } func (s *state) CountAgents() int { - return s.agents.CountAll() + return s.agents.Size() } func (s *state) CountClients() int { - return s.clients.CountAll() + return s.clients.Size() } func (s *state) CountIntercepts() int { - return s.intercepts.CountAll() + return s.intercepts.Size() } func (s *state) CountSessions() int { @@ -438,88 +439,81 @@ func (s *state) CountTunnelEgress() uint64 { // Sessions: Agents //////////////////////////////////////////////////////////////////////////////// -func (s *state) AddAgent(agent *rpc.AgentInfo, now time.Time) string { - s.mu.Lock() - defer s.mu.Unlock() - - sessionID := AgentSessionIDPrefix + uuid.New().String() +func (s *state) AddAgent(ctx context.Context, agent *rpc.AgentInfo, now time.Time) (string, error) { + if mutator.GetMap(ctx).IsInactive(agent.PodName) { + return "", status.Error(codes.Aborted, "inactivated pod") + } + sessionID := AgentSessionIDPrefix + agent.PodName + "." + agent.Namespace if oldAgent, hasConflict := s.agents.LoadOrStore(sessionID, agent); hasConflict { - panic(fmt.Errorf("duplicate id %q, existing %+v, new %+v", sessionID, oldAgent, agent)) + return "", status.Error(codes.AlreadyExists, fmt.Sprintf("duplicate id %q, existing %+v, new %+v", sessionID, oldAgent, agent)) } - agn, _ := s.agentsByName.LoadOrCompute(agent.Name, func() *xsync.MapOf[string, *rpc.AgentInfo] { - return xsync.NewMapOf[string, *rpc.AgentInfo]() - }) - agn.Store(sessionID, agent) s.sessions.Store(sessionID, newAgentSessionState(s.backgroundCtx, now)) - - for interceptID, intercept := range s.intercepts.LoadAll() { + s.intercepts.Range(func(interceptID string, intercept *rpc.InterceptInfo) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED { - continue + return true } // Check whether each intercept needs to either (1) be moved in to a NO_AGENT state // because this agent made things inconsistent, or (2) be moved out of a NO_AGENT // state because it just gained an agent. if errCode, errMsg := s.checkAgentsForIntercept(intercept); errCode != 0 { - intercept.Disposition = errCode - intercept.Message = errMsg - s.intercepts.Store(interceptID, intercept) + s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { + intercept.Disposition = errCode + intercept.Message = errMsg + }) } else if intercept.Disposition == rpc.InterceptDispositionType_NO_AGENT { - intercept.Disposition = rpc.InterceptDispositionType_WAITING - intercept.Message = "" - s.intercepts.Store(interceptID, intercept) + s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { + intercept.Disposition = rpc.InterceptDispositionType_WAITING + intercept.Message = "" + }) } - } - return sessionID + return true + }) + return sessionID, nil } func (s *state) GetAgent(sessionID string) *rpc.AgentInfo { - ret, _ := s.agents.Load(sessionID) - return ret -} - -func (s *state) GetActiveAgent(sessionID string) *rpc.AgentInfo { if ret, ok := s.agents.Load(sessionID); ok { - if as := s.GetSession(sessionID); as != nil && as.Active() { + if !mutator.GetMap(s.backgroundCtx).IsInactive(ret.PodName) { return ret } } return nil } -func (s *state) getAllAgents() map[string]*rpc.AgentInfo { - return s.agents.LoadAll() +func (s *state) EachAgent(f func(string, *rpc.AgentInfo) bool) { + m := mutator.GetMap(s.backgroundCtx) + s.agents.Range(func(si string, ag *rpc.AgentInfo) bool { + if !m.IsInactive(ag.PodName) { + return f(si, ag) + } + return true + }) } -func (s *state) HasAgent(name, namespace string) bool { - _, ok := s.agentsByName.Load(name) - return ok +func (s *state) LoadMatchingAgents(f func(string, *rpc.AgentInfo) bool) map[string]*rpc.AgentInfo { + m := mutator.GetMap(s.backgroundCtx) + return s.agents.LoadMatching(func(s string, ai *rpc.AgentInfo) bool { + return !m.IsInactive(ai.PodName) && f(s, ai) + }) } -func (s *state) getAgentsByName(name, namespace string) map[string]*rpc.AgentInfo { - agn, ok := s.agentsByName.Load(name) - if !ok { - return nil - } - ret := make(map[string]*rpc.AgentInfo, agn.Size()) - agn.Range(func(k string, v *rpc.AgentInfo) bool { - if v.Namespace == namespace { - ret[k] = proto.Clone(v).(*rpc.AgentInfo) +func (s *state) HasAgent(name, namespace string) (ok bool) { + s.EachAgent(func(_ string, ai *rpc.AgentInfo) bool { + if ai.Name == name && ai.Namespace == namespace { + ok = true + return false } return true }) - return ret + return ok } func (s *state) WatchAgents( ctx context.Context, filter func(sessionID string, agent *rpc.AgentInfo) bool, -) <-chan watchable.Snapshot[*rpc.AgentInfo] { - if filter == nil { - return s.agents.Subscribe(ctx) - } else { - return s.agents.SubscribeSubset(ctx, filter) - } +) <-chan map[string]*rpc.AgentInfo { + return s.agents.Subscribe(ctx.Done(), filter) } func (s *state) WatchWorkloads(ctx context.Context, ns string) (ch <-chan []workload.Event, err error) { @@ -540,8 +534,8 @@ func (s *state) WatchWorkloads(ctx context.Context, ns string) (ch <-chan []work // getAgentsInNamespace returns the session IDs the agents in the given namespace. func (s *state) getAgentsInNamespace(namespace string) map[string]*rpc.AgentInfo { - return s.agents.LoadAllMatching(func(_ string, ii *rpc.AgentInfo) bool { - return ii.Namespace == namespace + return s.LoadMatchingAgents(func(_ string, ai *rpc.AgentInfo) bool { + return ai.Namespace == namespace }) } @@ -572,29 +566,8 @@ func (s *state) UpdateIntercept(interceptID string, apply func(*rpc.InterceptInf } } -// UpdateClient applies a given mutator function to the stored client with sessionID; -// storing and returning the result. If the given client does not exist, then the mutator -// function is not run, and nil is returned. -func (s *state) UpdateClient(sessionID string, apply func(*rpc.ClientInfo)) *rpc.ClientInfo { - for { - cur, ok := s.clients.Load(sessionID) - if !ok || cur == nil { - // Doesn't exist (possibly was deleted while this loop was running). - return nil - } - - newInfo := proto.Clone(cur).(*rpc.ClientInfo) - apply(newInfo) - - swapped := s.clients.CompareAndSwap(sessionID, cur, newInfo) - if swapped { - return newInfo - } - } -} - func (s *state) RemoveIntercept(ctx context.Context, interceptID string) { - if intercept, didDelete := s.intercepts.LoadAndDelete(interceptID); didDelete { + if intercept, ok := s.intercepts.LoadAndDelete(interceptID); ok { s.FinalizeIntercept(ctx, intercept) } } @@ -613,8 +586,42 @@ func (s *state) DropIntercept(interceptID string) { s.intercepts.Delete(interceptID) } +func (s *state) UninstallAgents(ctx context.Context, ur *rpc.UninstallAgentsRequest) error { + sessionID := ur.GetSessionInfo().GetSessionId() + clientInfo := s.GetClient(sessionID) + if clientInfo == nil { + return status.Errorf(codes.NotFound, "Client session %q not found", sessionID) + } + ns := clientInfo.GetNamespace() + mm := mutator.GetMap(ctx) + agents := ur.Agents + if len(agents) == 0 { + if err := mm.DeleteAllPodsWithConfig(ctx, ns); err != nil { + return status.Errorf(codes.Internal, "unable to delete pods with agent: %v", err) + } + return nil + } + + wls := make([]k8sapi.Workload, len(agents)) + for i, agent := range agents { + wl, err := k8sapi.GetWorkload(ctx, agent, ns, "") + if err != nil { + return status.Errorf(codes.NotFound, "Workload %s.%s not found", agent, ns) + } + wls[i] = wl + } + + for _, wl := range wls { + mm.Delete(wl.GetName(), ns) + if err := mm.DeletePodsWithConfig(ctx, wl); err != nil { + return status.Errorf(codes.Internal, "unable to delete agent for workload %s.%s: %v", wl.GetName(), ns, err) + } + } + return nil +} + func (s *state) LoadMatchingIntercepts(filter func(string, *rpc.InterceptInfo) bool) map[string]*rpc.InterceptInfo { - return s.intercepts.LoadAllMatching(filter) + return s.intercepts.LoadMatching(filter) } func (s *state) GetIntercept(interceptID string) (*rpc.InterceptInfo, bool) { @@ -624,12 +631,8 @@ func (s *state) GetIntercept(interceptID string) (*rpc.InterceptInfo, bool) { func (s *state) WatchIntercepts( ctx context.Context, filter func(sessionID string, intercept *rpc.InterceptInfo) bool, -) <-chan watchable.Snapshot[*rpc.InterceptInfo] { - if filter == nil { - return s.intercepts.Subscribe(ctx) - } else { - return s.intercepts.SubscribeSubset(ctx, filter) - } +) <-chan map[string]*rpc.InterceptInfo { + return s.intercepts.Subscribe(ctx.Done(), filter) } func (s *state) Tunnel(ctx context.Context, stream tunnel.Stream) error { @@ -723,7 +726,7 @@ func (s *state) getAgentForDial(ctx context.Context, clientSessionID string, pod func (s *state) getAgentIdForDial(ctx context.Context, clientSessionID string, podIP netip.Addr) (string, error) { // An agent with a podIO matching the given podIP has precedence - agents := s.agents.LoadAllMatching(func(key string, ai *rpc.AgentInfo) bool { + agents := s.LoadMatchingAgents(func(key string, ai *rpc.AgentInfo) bool { if aip, err := netip.ParseAddr(ai.PodIp); err == nil { return podIP == aip } diff --git a/cmd/traffic/cmd/manager/state/state_test.go b/cmd/traffic/cmd/manager/state/state_test.go index 16284b94ad..1c527c7119 100644 --- a/cmd/traffic/cmd/manager/state/state_test.go +++ b/cmd/traffic/cmd/manager/state/state_test.go @@ -7,12 +7,16 @@ import ( "github.com/puzpuzpuz/xsync/v3" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/rpc/v2/manager" + "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/mutator" testdata "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/test" + "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/watchable" "github.com/telepresenceio/telepresence/v2/pkg/log" + "github.com/telepresenceio/telepresence/v2/pkg/workload" ) type suiteState struct { @@ -25,12 +29,15 @@ type suiteState struct { func (s *suiteState) SetupTest() { s.ctx = dlog.NewTestContext(s.T(), false) s.state = &state{ - backgroundCtx: s.ctx, - sessions: xsync.NewMapOf[string, SessionState](), - agentsByName: xsync.NewMapOf[string, *xsync.MapOf[string, *manager.AgentInfo]](), - interceptStates: xsync.NewMapOf[string, *interceptState](), - timedLogLevel: log.NewTimedLevel("debug", log.SetLevel), - llSubs: newLoglevelSubscribers(), + backgroundCtx: s.ctx, + intercepts: watchable.NewMap[string, *manager.InterceptInfo](interceptEqual, time.Millisecond), + agents: watchable.NewMap[string, *manager.AgentInfo](agentsEqual, time.Millisecond), + clients: xsync.NewMapOf[string, *manager.ClientInfo](), + workloadWatchers: xsync.NewMapOf[string, workload.Watcher](), + sessions: xsync.NewMapOf[string, SessionState](), + interceptStates: xsync.NewMapOf[string, *interceptState](), + timedLogLevel: log.NewTimedLevel("debug", log.SetLevel), + llSubs: newLoglevelSubscribers(), } } @@ -44,6 +51,25 @@ func (fc *FakeClock) Now() time.Time { return base.Add(offset) } +func getAllAgents(st *state) []*manager.AgentInfo { + agents := make([]*manager.AgentInfo, 0, st.agents.Size()) + st.agents.Range(func(_ string, a *manager.AgentInfo) bool { + agents = append(agents, a) + return true + }) + return agents +} + +func getAgentsByName(st *state, name, namespace string) (agents []*manager.AgentInfo) { + st.agents.Range(func(_ string, a *manager.AgentInfo) bool { + if a.Name == name && a.Namespace == namespace { + agents = append(agents, a) + } + return true + }) + return agents +} + func (s *suiteState) TestStateInternal() { ctx := context.Background() @@ -59,42 +85,48 @@ func (s *suiteState) TestStateInternal() { demoAgent2 := testAgents["demo2"] clock := &FakeClock{} - s := NewState(ctx).(*state) - - h := s.AddAgent(helloAgent, clock.Now()) - hp := s.AddAgent(helloProAgent, clock.Now()) - d1 := s.AddAgent(demoAgent1, clock.Now()) - d2 := s.AddAgent(demoAgent2, clock.Now()) - - a.Equal(helloAgent, s.GetAgent(h)) - a.Equal(helloProAgent, s.GetAgent(hp)) - a.Equal(demoAgent1, s.GetAgent(d1)) - a.Equal(demoAgent2, s.GetAgent(d2)) - - agents := s.getAllAgents() + m := mutator.NewWatcher() + ctx = mutator.WithMap(ctx, m) + st := NewState(ctx).(*state) + + h, err := st.AddAgent(ctx, helloAgent, clock.Now()) + require.NoError(t, err) + hp, err := st.AddAgent(ctx, helloProAgent, clock.Now()) + require.NoError(t, err) + d1, err := st.AddAgent(ctx, demoAgent1, clock.Now()) + require.NoError(t, err) + d2, err := st.AddAgent(ctx, demoAgent2, clock.Now()) + require.NoError(t, err) + + a.Equal(helloAgent, st.GetAgent(h)) + a.Equal(helloProAgent, st.GetAgent(hp)) + a.Equal(demoAgent1, st.GetAgent(d1)) + a.Equal(demoAgent2, st.GetAgent(d2)) + + agents := getAllAgents(st) a.Len(agents, 4) a.Contains(agents, helloAgent) a.Contains(agents, helloProAgent) a.Contains(agents, demoAgent1) a.Contains(agents, demoAgent2) - agents = s.getAgentsByName("hello", "default") + agents = getAgentsByName(st, "hello", "default") a.Len(agents, 1) a.Contains(agents, helloAgent) - agents = s.getAgentsByName("hello-pro", "default") + agents = getAgentsByName(st, "hello-pro", "default") a.Len(agents, 1) a.Contains(agents, helloProAgent) - agents = s.getAgentsByName("demo", "default") + agents = getAgentsByName(st, "demo", "default") a.Len(agents, 2) a.Contains(agents, demoAgent1) a.Contains(agents, demoAgent2) - agents = s.getAgentsByName("does-not-exist", "default") + agents = getAgentsByName(st, "does-not-exist", "default") a.Len(agents, 0) - agents = s.getAgentsByName("hello", "does-not-exist") + agents = getAgentsByName(st, "hello", "does-not-exist") a.Len(agents, 0) }) diff --git a/cmd/traffic/cmd/manager/state/workload_info_watcher.go b/cmd/traffic/cmd/manager/state/workload_info_watcher.go index 65d059128b..e2a39b2dc2 100644 --- a/cmd/traffic/cmd/manager/state/workload_info_watcher.go +++ b/cmd/traffic/cmd/manager/state/workload_info_watcher.go @@ -12,6 +12,7 @@ import ( "github.com/datawire/dlib/dlog" rpc "github.com/telepresenceio/telepresence/rpc/v2/manager" + "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/mutator" "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" "github.com/telepresenceio/telepresence/v2/pkg/workload" @@ -99,14 +100,14 @@ func (wf *workloadInfoWatcher) Watch(ctx context.Context, stream rpc.Manager_Wat dlog.Debug(ctx, "Agents channel closed") return nil } - wf.handleAgentSnapshot(ctx, ais.State) + wf.handleAgentSnapshot(ctx, ais) // Events that arrive at the intercept channel should be counted as modifications. case is, ok := <-interceptsCh: if !ok { dlog.Debug(ctx, "Intercepts channel closed") return nil } - wf.handleInterceptSnapshot(ctx, is.State) + wf.handleInterceptSnapshot(ctx, is) } } } @@ -258,8 +259,10 @@ func (wf *workloadInfoWatcher) handleWorkloadsSnapshot(ctx context.Context, wes func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[string]*rpc.AgentInfo) { oldAgentInfos := wf.agentInfos wf.agentInfos = ais + m := mutator.GetMap(ctx) for k, a := range oldAgentInfos { - if _, ok := ais[k]; !ok { + ai, ok := ais[k] + if !ok || m.IsInactive(ai.PodName) { name := a.Name as := rpc.WorkloadInfo_NO_AGENT_UNSPECIFIED if w, ok := wf.workloadEvents[name]; ok && w.Type != rpc.WorkloadEvent_DELETED { @@ -289,6 +292,9 @@ func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[ } } for _, a := range ais { + if m.IsInactive(a.PodName) { + continue + } name := a.Name var iClients []*rpc.WorkloadInfo_Intercept as := rpc.WorkloadInfo_INSTALLED diff --git a/cmd/traffic/cmd/manager/test/testdata/agents.yaml b/cmd/traffic/cmd/manager/test/testdata/agents.yaml index 2d2ee6a719..b122f683fd 100644 --- a/cmd/traffic/cmd/manager/test/testdata/agents.yaml +++ b/cmd/traffic/cmd/manager/test/testdata/agents.yaml @@ -2,6 +2,7 @@ hello: name: hello namespace: default + pod_name: hello-3487fcx0-uxhgh hostname: hello-abcdef-123 product: telepresence-agent version: "1" @@ -12,6 +13,7 @@ hello: helloPro: name: hello-pro namespace: default + pod_name: hello-pro-f4784865-wgzxg hostname: hello-pro-abcdef-123 product: telepresence-agent version: "1" @@ -28,6 +30,7 @@ helloPro: demo1: name: demo namespace: default + pod_name: demo-e2784865-xgamd hostname: demo-abcdef-123 product: telepresence-agent version: "1" @@ -44,6 +47,7 @@ demo1: demo2: name: demo namespace: default + pod_name: demo-33fce865-bgdxf hostname: demo-abcdef-456 product: telepresence-agent version: "1" diff --git a/cmd/traffic/cmd/manager/watchable/map.go b/cmd/traffic/cmd/manager/watchable/map.go index 7ab6f91eed..6c2865537b 100644 --- a/cmd/traffic/cmd/manager/watchable/map.go +++ b/cmd/traffic/cmd/manager/watchable/map.go @@ -1,411 +1,241 @@ package watchable import ( - "context" - "sync" + "math" + "sync/atomic" + "time" - "google.golang.org/protobuf/proto" - - "github.com/telepresenceio/telepresence/v2/pkg/maps" + "github.com/google/uuid" + "github.com/puzpuzpuz/xsync/v3" ) -type Message = proto.Message - -// Update describes a mutation made to a Map. -type Update[V Message] struct { - Key string - Delete bool // Whether this is deleting the entry for .Key, or setting it to .Value. - Value V -} - -// Snapshot contains a snapshot of the current state of a Map, as well as a list of -// changes that have happened since the last snapshot. -type Snapshot[V Message] struct { - // State is the current state of the snapshot. - State map[string]V - // Updates is the list of mutations that have happened since the previous snapshot. - // Mutations that delete a value have .Delete=true, and .Value set to the value that was - // deleted. No-op updates are not included (i.e., setting something to its current value, - // or deleting something that does not exist). - Updates []Update[V] +type subscription[K comparable, V any] struct { + channel chan map[K]V + filter func(K, V) bool + mark atomic.Bool + doneCh <-chan struct{} } -// Map is a wrapper around map[string]VALTYPE that is very similar to sync.Map, and that -// provides the additional features that: -// -// 1. it is thread-safe (compared to a bare map) -// 2. it provides type safety (compared to a sync.Map) -// 3. it provides a compare-and-swap operation -// 4. you can Subscribe to either the whole map or just a subset of the map to watch for updates. -// This gives you complete snapshots, deltas, and coalescing of rapid updates. -type Map[V Message] struct { - lock sync.RWMutex - // things guarded by 'lock' - close chan struct{} // can read from the channel while unlocked, IF you've already validated it's non-nil - value map[string]V - subscribers map[<-chan Update[V]]chan<- Update[V] // readEnd ↦ writeEnd - - // not guarded by 'lock' - wg sync.WaitGroup +type Map[K comparable, V any] struct { + *xsync.MapOf[K, V] + equal func(V, V) bool + subscribers *xsync.MapOf[uuid.UUID, *subscription[K, V]] + notifyDelay time.Duration + notifier *time.Timer } -func (tm *Map[V]) unlockedInit() { - if tm.close == nil { - tm.close = make(chan struct{}) - tm.value = make(map[string]V) - tm.subscribers = make(map[<-chan Update[V]]chan<- Update[V]) +// NewMap creates a new Map instance configured with the given options. +func NewMap[K comparable, V any](equal func(V, V) bool, notifyDelay time.Duration, config ...func(*xsync.MapConfig)) *Map[K, V] { + m := &Map[K, V]{ + MapOf: xsync.NewMapOf[K, V](config...), + equal: equal, + subscribers: xsync.NewMapOf[uuid.UUID, *subscription[K, V]](), + notifyDelay: notifyDelay, } + m.notifier = time.AfterFunc(math.MaxInt64, m.notify) + return m } -func (tm *Map[V]) unlockedIsClosed() bool { +// Subscribe returns a channel that will emit a snapshot of the map that corresponds to the content +// of the map filtered by the given filter. The filter is re-evaluated each time a snapshot is +// emitted. +// +// The first snapshot is emitted immediately after the call to Subscribe(), and then whenever the map +// changes a key - value binding for which the filter evaluates to true. +// +// The snapshot content will reflect actual values in the map. Mutating them will thus mutate the +// map without the map's knowledge, and hence not trigger notifications to subscribers. +// +// The returned channel will be closed when the given channel is closed. +func (m *Map[K, V]) Subscribe(done <-chan struct{}, filter func(K, V) bool) <-chan map[K]V { + ch := make(chan map[K]V, 1) select { - case <-tm.close: - return true + case <-done: + close(ch) default: - return false - } -} - -func (tm *Map[V]) unlockedLoadAll() map[string]V { - ret := make(map[string]V, len(tm.value)) - for k, v := range tm.value { - ret[k] = proto.Clone(v).(V) - } - return ret -} - -// LoadAll returns a deepcopy of all key/value pairs in the map. -func (tm *Map[V]) LoadAll() map[string]V { - tm.lock.RLock() - defer tm.lock.RUnlock() - return tm.unlockedLoadAll() -} - -// CountAll returns a count of all key/value pairs in the map. -func (tm *Map[V]) CountAll() int { - tm.lock.RLock() - defer tm.lock.RUnlock() - return len(tm.value) -} - -// LoadAllMatching returns a deepcopy of all key/value pairs in the map for which the given -// function returns true. The map is locked during the evaluation of the filter. -func (tm *Map[V]) LoadAllMatching(filter func(string, V) bool) map[string]V { - tm.lock.RLock() - defer tm.lock.RUnlock() - ret := make(map[string]V) - for k, v := range tm.value { - if filter(k, v) { - ret[k] = proto.Clone(v).(V) + id := uuid.New() + if filter == nil { + filter = func(K, V) bool { return true } } + sb := &subscription[K, V]{filter: filter, channel: ch} + m.subscribers.Store(id, sb) + go func() { + // Trigger initial snapshot, then wait for subscription to end + m.sendSnapshot(sb) + <-done + m.subscribers.Delete(id) + close(ch) + }() } - return ret + return ch } -// Load returns a deepcopy of the value for a specific key. -func (tm *Map[V]) Load(key string) (value V, ok bool) { - tm.lock.RLock() - defer tm.lock.RUnlock() - ret, ok := tm.value[key] - if !ok { - return ret, false +// Compute either sets the computed new value for the key or deletes the value for the key. +// When the delete result of the valueFn function is set to true, the value will be deleted if it exists. +// When delete is set to false, the value is updated to the newValue. The ok result indicates whether the +// value was computed and stored, thus, is present in the map. The actual result contains the new value in +// cases where the value was computed and stored. See the example for a few use cases. +// +// This call locks a hash table bucket while the compute function is executed. It means that modifications +// on other entries in the bucket will be blocked until the valueFn executes. Consider this when the function +// includes long-running operations. +func (m *Map[K, V]) Compute(key K, f func(oldValue V, loaded bool) (newValue V, drop bool)) (actual V, ok bool) { + didMark := false + actual, ok = m.MapOf.Compute(key, func(oldValue V, loaded bool) (V, bool) { + value, del := f(oldValue, loaded) + if del { + if loaded && m.markSubscribers(key, oldValue) { + didMark = true + } + } else if !(loaded && m.equal(value, oldValue)) && m.markSubscribers(key, value) { + didMark = true + } + return value, del + }) + if didMark { + m.notifier.Reset(m.notifyDelay) } - return proto.Clone(ret).(V), true + return actual, ok } -// Store sets a key sets the value for a key. This blocks forever if .Close() has already been -// called. -func (tm *Map[V]) Store(key string, val V) { - tm.lock.Lock() - defer tm.lock.Unlock() - - tm.unlockedStore(key, val) +// CompareAndSwap checks if the current value for the given key equals the oldValue, and if +// so, swaps the current value for the newValue. +// The swapped result reports whether the value was swapped. +func (m *Map[K, V]) CompareAndSwap(key K, oldValue, newValue V) (swapped bool) { + m.Compute(key, func(cur V, loaded bool) (V, bool) { + if loaded && m.equal(cur, oldValue) { + swapped = true + return newValue, false + } + return oldValue, !loaded + }) + return swapped } -// LoadOrStore returns the existing value for the key if present. Otherwise, it stores and returns -// the given value. The 'loaded' result is true if the value was loaded, false if stored. -// -// If the value does need to be stored, all the same blocking semantics as .Store() apply. -func (tm *Map[V]) LoadOrStore(key string, val V) (value V, loaded bool) { - tm.lock.Lock() - defer tm.lock.Unlock() - - loadedVal, loadedOK := tm.value[key] - if loadedOK { - return proto.Clone(loadedVal).(V), true - } - tm.unlockedStore(key, val) - return proto.Clone(val).(V), false +// Delete deletes the value for a key, returning the previous value if any. +// The loaded result reports whether the key was present. +func (m *Map[K, V]) Delete(key K) { + m.Compute(key, func(oldValue V, wasLoaded bool) (V, bool) { return oldValue, true }) } -// CompareAndSwap is the atomic equivalent of: -// -// if loadedVal, loadedOK := m.Load(key); loadedOK && proto.Equal(loadedVal, old) { -// m.Store(key, new) -// return true -// } -// return false -func (tm *Map[V]) CompareAndSwap(key string, oldVal, newVal V) bool { - tm.lock.Lock() - defer tm.lock.Unlock() - - if loadedVal, loadedOK := tm.value[key]; loadedOK && proto.Equal(loadedVal, oldVal) { - tm.unlockedStore(key, newVal) +// LoadAll return a map of all entries. +func (m *Map[K, V]) LoadAll() map[K]V { + mr := make(map[K]V, m.Size()) + m.Range(func(key K, value V) bool { + mr[key] = value return true - } - return false + }) + return mr } -func (tm *Map[V]) unlockedStore(key string, val V) { - tm.unlockedInit() - if tm.unlockedIsClosed() { - // block forever - tm.lock.Unlock() - select {} - } - - tm.value[key] = val - for _, subscriber := range tm.subscribers { - subscriber <- Update[V]{ - Key: key, - Value: proto.Clone(val).(V), +// LoadMatching return a map of all entries matching the given filter. +func (m *Map[K, V]) LoadMatching(filter func(K, V) bool) map[K]V { + mr := make(map[K]V) + m.Range(func(key K, value V) bool { + if filter(key, value) { + mr[key] = value } - } + return true + }) + return mr } -// Delete deletes the value for a key. This blocks forever if .Close() has already been called. -func (tm *Map[V]) Delete(key string) { - tm.lock.Lock() - defer tm.lock.Unlock() - - tm.unlockedDelete(key) +// LoadAndDelete deletes the value for a key, returning the previous value if any. +// The loaded result reports whether the key was present. +func (m *Map[K, V]) LoadAndDelete(key K) (previous V, loaded bool) { + m.Compute(key, func(oldValue V, wasLoaded bool) (V, bool) { + if wasLoaded { + previous = oldValue + loaded = true + } + return oldValue, true + }) + return previous, loaded } -func (tm *Map[V]) unlockedDelete(key string) { - tm.unlockedInit() - if tm.unlockedIsClosed() { - // block forever - tm.lock.Unlock() - select {} - } - - if tm.value == nil { - return - } - delete(tm.value, key) - for _, subscriber := range tm.subscribers { - subscriber <- Update[V]{ - Key: key, - Delete: true, +// LoadAndStore stores a new value for the key and returns the existing one, if present. The loaded result is true if the +// existing value was loaded, false otherwise. +func (m *Map[K, V]) LoadAndStore(key K, value V) (existing V, loaded bool) { + m.Compute(key, func(oldValue V, wasLoaded bool) (V, bool) { + if wasLoaded { + existing = oldValue + loaded = true } - } + return value, false + }) + return existing, loaded } -// LoadAndDelete deletes the value for a key, returning a deepcopy of the previous value if any. -// The 'loaded' result reports whether the key was present. +// LoadOrCompute returns the existing value for the key if present. Otherwise, it computes the value using +// the provided function and returns the computed value. The loaded result is true if the value was loaded, +// false if stored. // -// If the value does need to be deleted, all the same blocking semantics as .Delete() apply. -func (tm *Map[V]) LoadAndDelete(key string) (value V, loaded bool) { - tm.lock.Lock() - defer tm.lock.Unlock() - - loadedVal, loadedOK := tm.value[key] - if !loadedOK { - return loadedVal, false - } - - tm.unlockedDelete(key) - - return proto.Clone(loadedVal).(V), true +// This call locks a hash table bucket while the compute function is executed. It means that modifications +// on other entries in the bucket will be blocked until the valueFn executes. Consider this when the function +// includes long-running operations. +func (m *Map[K, V]) LoadOrCompute(key K, valueFn func() V) (actual V, loaded bool) { + actual, _ = m.Compute(key, func(oldValue V, wasLoaded bool) (V, bool) { + if wasLoaded { + loaded = true + return oldValue, false + } + return valueFn(), false + }) + return actual, loaded } -// Close marks the map as "finished", all subscriber channels are closed and further mutations are -// forbidden. -// -// After .Close() is called, any calls to .Store() will block forever, and any calls to .Subscribe() -// will return an already-closed channel. -// -// .Load() and .LoadAll() calls will continue to work normally after .Close() has been called. -func (tm *Map[V]) Close() { - tm.lock.Lock() - - tm.unlockedInit() - if !tm.unlockedIsClosed() { - close(tm.close) - } - tm.lock.Unlock() - tm.wg.Wait() +// LoadOrStore returns the existing value for the key if present. Otherwise, it stores and returns the given value. +// The loaded result is true if the value was loaded, false if stored. +func (m *Map[K, V]) LoadOrStore(key K, value V) (actual V, loaded bool) { + actual, _ = m.Compute(key, func(oldValue V, wasLoaded bool) (V, bool) { + if wasLoaded { + loaded = true + return oldValue, false + } + return value, false + }) + return actual, loaded } -// internalSubscribe returns a channel (that blocks on both ends), that is written to on each map -// update. If the map is already Close()ed, then this returns nil. -func (tm *Map[V]) internalSubscribe(_ context.Context) (<-chan Update[V], map[string]V) { - tm.lock.Lock() - defer tm.lock.Unlock() - tm.unlockedInit() - - ret := make(chan Update[V]) - if tm.unlockedIsClosed() { - return nil, nil - } - tm.subscribers[ret] = ret - return ret, tm.unlockedLoadAll() +// Store stores a new value for the key. +func (m *Map[K, V]) Store(key K, value V) { + m.Compute(key, func(oldValue V, wasLoaded bool) (V, bool) { return value, false }) } -// Subscribe returns a channel that will emit a complete snapshot of the map immediately after the -// call to Subscribe(), and then whenever the map changes. Updates are coalesced; if you do not -// need to worry about reading from the channel faster than you are able. The snapshot will contain -// the full list of coalesced updates; the initial snapshot will contain 0 updates. A read from the -// channel will block as long as there are no changes since the last read. -// -// The values in the snapshot are deepcopies of the actual values in the map, but values may be -// reused between snapshots; if you mutate a value in a snapshot, that mutation may erroneously -// persist in future snapshots. -// -// The returned channel will be closed when the Context is Done, or .Close() is called. If .Close() -// has already been called, then an already-closed channel is returned. -func (tm *Map[V]) Subscribe(ctx context.Context) <-chan Snapshot[V] { - return tm.SubscribeSubset(ctx, func(string, V) bool { +// markSubscribers marks all subscribers interested in the given key and value binding. +func (m *Map[K, V]) markSubscribers(key K, value V) (didMark bool) { + m.subscribers.Range(func(_ uuid.UUID, sb *subscription[K, V]) bool { + if !sb.mark.Load() && sb.filter(key, value) { + didMark = true + sb.mark.Store(true) + } return true }) + return didMark } -// SubscribeSubset is like Subscribe, but the snapshot returned only includes entries that satisfy -// the 'include' predicate. Mutations to entries that don't satisfy the predicate do not cause a -// new snapshot to be emitted. If the value for a key changes from satisfying the predicate to not -// satisfying it, then this is treated as a delete operation, and a new snapshot is generated. -func (tm *Map[V]) SubscribeSubset(ctx context.Context, include func(string, V) bool) <-chan Snapshot[V] { - upstream, initialSnapshot := tm.internalSubscribe(ctx) - downstream := make(chan Snapshot[V]) - - if upstream == nil { - close(downstream) - return downstream - } - - tm.wg.Add(1) - go tm.coalesce(ctx, include, upstream, downstream, initialSnapshot) - - return downstream -} - -func (tm *Map[V]) coalesce( - ctx context.Context, - includep func(string, V) bool, - upstream <-chan Update[V], - downstream chan<- Snapshot[V], - initialSnapshot map[string]V, -) { - defer tm.wg.Done() - defer close(downstream) - - var shutdown func() - shutdown = func() { - shutdown = func() {} // Make this function an empty one after first run to prevent calling the following goroutine multiple times - // Do this asynchronously because getting the lock might block a .Store() that's - // waiting on us to read from 'upstream'! We don't need to worry about separately - // waiting for this goroutine because we implicitly do that when we drain - // 'upstream'. - go func() { - tm.lock.Lock() - defer tm.lock.Unlock() - close(tm.subscribers[upstream]) - delete(tm.subscribers, upstream) - }() - } - - // Cur is a snapshot of the current state all the map according to all MAPTYPEUpdates we've - // received from 'upstream', with any entries removed that do not satisfy the predicate - // 'includep'. - cur := make(map[string]V) - for k, v := range initialSnapshot { - if includep(k, v) { - cur[k] = v +// notify will send a snapshot to all subscribers that have been marked. +func (m *Map[K, V]) notify() { + m.subscribers.Range(func(_ uuid.UUID, sb *subscription[K, V]) bool { + if sb.mark.CompareAndSwap(true, false) { + m.sendSnapshot(sb) } - } - - snapshot := Snapshot[V]{ - // snapshot.State is a copy of 'cur' that we send to the 'downstream' channel. We - // don't send 'cur' directly because we're necessarily in a separate goroutine from - // the reader of 'downstream', and map gets/sets aren't thread-safe, so we'd risk - // memory corruption with our updating of 'cur' and the reader's accessing of 'cur'. - // snapshot.State gets set to 'nil' when we need to do a read before we can write to - // 'downstream' again. - State: maps.Copy(cur), - Updates: nil, - } - - // applyUpdate applies an update to 'cur', and updates 'snapshot.State' as nescessary. - applyUpdate := func(update Update[V]) { - if update.Delete || !includep(update.Key, update.Value) { - if old, haveOld := cur[update.Key]; haveOld { - update.Delete = true - update.Value = old - snapshot.Updates = append(snapshot.Updates, update) - delete(cur, update.Key) - if snapshot.State != nil { - delete(snapshot.State, update.Key) - } else { - snapshot.State = make(map[string]V, len(cur)) - for k, v := range cur { - snapshot.State[k] = v - } - } - } - } else { - if old, haveOld := cur[update.Key]; !haveOld || !proto.Equal(old, update.Value) { - snapshot.Updates = append(snapshot.Updates, update) - cur[update.Key] = update.Value - if snapshot.State != nil { - snapshot.State[update.Key] = update.Value - } else { - snapshot.State = maps.Copy(cur) - } - } - } - } + return true + }) +} - // The following loop is reading both a tm.close channel and the ctx.Done() channel. When the - // tm.close channel is closed, the Map as a whole has been closed, and when ctx.Done() is closed, - // the subscription that started this call to coalesce has ended. If one of the channels close, - // the loop must call shutdown() and then continue looping, now in a way that never selects the - // closed channel. The closed channel is therefore set to `nil` so that it blocks forever, which - // in essence means that the only way out of the loop is to close the `upstream` channel. This - // happens when the subscription ends. - closeCh := tm.close - doneCh := ctx.Done() - for { - if snapshot.State == nil { - select { - case <-doneCh: - shutdown() - doneCh = nil - case <-closeCh: - shutdown() - closeCh = nil - case update, readOK := <-upstream: - if !readOK { - return - } - applyUpdate(update) - } - } else { - // Same as above, but with an additional "downstream <- snapshot" case. - select { - case <-doneCh: - shutdown() - doneCh = nil - case <-closeCh: - shutdown() - closeCh = nil - case update, readOK := <-upstream: - if !readOK { - return - } - applyUpdate(update) - case downstream <- snapshot: - snapshot = Snapshot[V]{} - } +// sendSnapshot evaluates and sends a snapshot to the subscriber. +func (m *Map[K, V]) sendSnapshot(sb *subscription[K, V]) { + snap := make(map[K]V) + m.Range(func(key K, value V) bool { + if sb.filter(key, value) { + snap[key] = value } + return true + }) + select { + case <-sb.doneCh: + case sb.channel <- snap: + default: } } diff --git a/cmd/traffic/cmd/manager/watchable/map_test.go b/cmd/traffic/cmd/manager/watchable/map_test.go deleted file mode 100644 index c1ddcde188..0000000000 --- a/cmd/traffic/cmd/manager/watchable/map_test.go +++ /dev/null @@ -1,426 +0,0 @@ -package watchable_test - -import ( - "context" - "testing" - "time" - - "github.com/go-json-experiment/json" - "github.com/go-json-experiment/json/jsontext" - "github.com/stretchr/testify/assert" - - "github.com/datawire/dlib/dlog" - "github.com/telepresenceio/telepresence/rpc/v2/manager" - "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/watchable" -) - -func assertMessageMapSnapshotEqual[V watchable.Message](t *testing.T, expected, actual watchable.Snapshot[V], msgAndArgs ...any) bool { - t.Helper() - - expectedBytes, err := json.Marshal(expected, jsontext.WithIndent(" "), json.Deterministic(true)) - if err != nil { - t.Fatal(err) - } - - actualBytes, err := json.Marshal(actual, jsontext.WithIndent(" "), json.Deterministic(true)) - if err != nil { - t.Fatal(err) - } - - if !assert.Equal(t, string(expectedBytes), string(actualBytes)) { - return false - } - - for k := range actual.State { - if !assertDeepCopies(t, expected.State[k], actual.State[k], msgAndArgs...) { - return false - } - } - - for i := range actual.Updates { - var m watchable.Message = expected.Updates[i].Value - if m == nil { - continue - } - if !assertDeepCopies(t, expected.Updates[i].Value, actual.Updates[i].Value, msgAndArgs...) { - return false - } - } - - return true -} - -func TestMessageMap_Close(t *testing.T) { - // TODO -} - -func agentInfoCtor(n string) *manager.AgentInfo { - return &manager.AgentInfo{Name: n} -} - -func agentInfoCmp(a *manager.AgentInfo, n string) bool { - return a.Name == n -} - -func TestMessageMap_Delete(t *testing.T) { - typedTestMessageMap_Delete[*manager.AgentInfo](t, agentInfoCtor) -} - -func typedTestMessageMap_Delete[V watchable.Message](t *testing.T, ctor func(string) V) { - var m watchable.Map[V] - - // Check that a delete on a zero map works - m.Delete("a") - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{State: map[string]V{}}, - watchable.Snapshot[V]{State: m.LoadAll()}) - - // Check that a normal delete works - m.Store("a", ctor("a")) - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{ - "a": ctor("a"), - }, - }, - watchable.Snapshot[V]{State: m.LoadAll()}) - m.Delete("a") - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{}, - }, - watchable.Snapshot[V]{State: m.LoadAll()}) - - // Check that a repeated delete works - m.Delete("a") - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{}, - }, - watchable.Snapshot[V]{State: m.LoadAll()}) -} - -func TestMessageMap_Load(t *testing.T) { - typedTestMessageMap_Load[*manager.AgentInfo](t, agentInfoCtor) -} - -func typedTestMessageMap_Load[V watchable.Message](t *testing.T, ctor func(string) V) { - var m watchable.Map[V] - - a := ctor("value") - m.Store("k", a) - - // Check that a load returns a copy of the input object - b, ok := m.Load("k") - assert.True(t, ok) - assertDeepCopies(t, a, b) - m.Delete("k") - - // Check that a load returns nil after a delete - c, ok := m.Load("k") - assert.False(t, ok) - assert.Nil(t, c) - - // Check that two sequential loads return distinct copies - m.Store("k", a) - d, ok := m.Load("k") - assert.True(t, ok) - e, ok := m.Load("k") - assert.True(t, ok) - assertDeepCopies(t, a, d) - assertDeepCopies(t, a, e) - assertDeepCopies(t, d, e) -} - -func TestMessageMap_LoadAll(t *testing.T) { - // TODO -} - -func TestMessageMap_LoadAndDelete(t *testing.T) { - typedTestMessageMap_LoadAndDelete[*manager.AgentInfo](t, agentInfoCtor) -} - -func typedTestMessageMap_LoadAndDelete[V watchable.Message](t *testing.T, ctor func(string) V) { - var m watchable.Map[V] - - a := ctor("value") - m.Store("k", a) - - // Check that a load returns a copy of the input object - b, ok := m.LoadAndDelete("k") - assert.True(t, ok) - assertDeepCopies(t, a, b) - - // Check that a load returns nil after a delete - c, ok := m.Load("k") - assert.False(t, ok) - assert.Nil(t, c) - - // Now check the non-existing case - d, ok := m.LoadAndDelete("k") - assert.False(t, ok) - assert.Nil(t, d) -} - -func TestMessageMap_LoadOrStore(t *testing.T) { - typedTestMessageMap_LoadOrStore[*manager.AgentInfo](t, agentInfoCtor) -} - -func typedTestMessageMap_LoadOrStore[V watchable.Message](t *testing.T, ctor func(string) V) { - var m watchable.Map[V] - - a := ctor("value") - m.Store("k", a) - - b := ctor("value") - assertDeepCopies(t, a, b) - - c, ok := m.LoadOrStore("k", b) - assert.True(t, ok) - assertDeepCopies(t, a, c) - assertDeepCopies(t, b, c) - - d, ok := m.LoadOrStore("k", b) - assert.True(t, ok) - assertDeepCopies(t, a, d) - assertDeepCopies(t, b, d) - assertDeepCopies(t, c, d) - - e, ok := m.LoadOrStore("x", a) - assert.False(t, ok) - assertDeepCopies(t, a, e) - assertDeepCopies(t, b, e) - assertDeepCopies(t, c, e) - assertDeepCopies(t, d, e) -} - -func TestMessageMap_Store(t *testing.T) { - // TODO -} - -func TestMessageMap_CompareAndSwap(t *testing.T) { - // TODO -} - -func TestMessageMap_Subscribe(t *testing.T) { - typedTestMessageMap_Subscribe[*manager.AgentInfo](t, agentInfoCtor) -} - -func typedTestMessageMap_Subscribe[V watchable.Message](t *testing.T, ctor func(string) V) { - ctx := dlog.NewTestContext(t, true) - ctx, cancelCtx := context.WithCancel(ctx) - var m watchable.Map[V] - - m.Store("a", ctor("A")) - m.Store("b", ctor("B")) - m.Store("c", ctor("C")) - - ch := m.Subscribe(ctx) - - // Check that a complete snapshot is immediately available - snapshot, ok := <-ch - assert.True(t, ok) - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{ - "a": ctor("A"), - "b": ctor("B"), - "c": ctor("C"), - }, - Updates: nil, - }, - snapshot) - - // Check that writes don't block on the subscriber channel - m.Store("d", ctor("D")) - m.Store("e", ctor("E")) - m.Store("f", ctor("F")) - - // Check that those 3 updates get coalesced in to a single read - snapshot, ok = <-ch - assert.True(t, ok) - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{ - "a": ctor("A"), - "b": ctor("B"), - "c": ctor("C"), - "d": ctor("D"), - "e": ctor("E"), - "f": ctor("F"), - }, - Updates: []watchable.Update[V]{ - {Key: "d", Value: ctor("D")}, - {Key: "e", Value: ctor("E")}, - {Key: "f", Value: ctor("F")}, - }, - }, - snapshot) - - // Check that deletes work - m.Delete("a") - snapshot, ok = <-ch - assert.True(t, ok) - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{ - "b": ctor("B"), - "c": ctor("C"), - "d": ctor("D"), - "e": ctor("E"), - "f": ctor("F"), - }, - Updates: []watchable.Update[V]{ - {Key: "a", Delete: true, Value: ctor("A")}, - }, - }, - snapshot) - - // Check that deletes work with LoadAndDelete - m.LoadAndDelete("b") - snapshot, ok = <-ch - assert.True(t, ok) - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{ - "c": ctor("C"), - "d": ctor("D"), - "e": ctor("E"), - "f": ctor("F"), - }, - Updates: []watchable.Update[V]{ - {Key: "b", Delete: true, Value: ctor("B")}, - }, - }, - snapshot) - - // Check that deletes coalesce with update - m.Store("c", ctor("c")) - m.Delete("c") - snapshot, ok = <-ch - assert.True(t, ok) - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{ - "d": ctor("D"), - "e": ctor("E"), - "f": ctor("F"), - }, - Updates: []watchable.Update[V]{ - {Key: "c", Value: ctor("c")}, - {Key: "c", Delete: true, Value: ctor("c")}, - }, - }, - snapshot) - - // Add some more writes, then close it - m.Store("g", ctor("G")) - m.Store("h", ctor("H")) - m.Store("i", ctor("I")) - cancelCtx() - // Because the 'close' happens asynchronously when the context ends, we need to wait a - // moment to ensure that it's actually closed before we hit the next step. - time.Sleep(500 * time.Millisecond) - - // Check that the writes get coalesced in to a "close". - snapshot, ok = <-ch - assert.False(t, ok) - assert.Zero(t, snapshot) - - snapshot, ok = <-ch - assert.False(t, ok) - assert.Zero(t, snapshot) - - snapshot, ok = <-ch - assert.False(t, ok) - assert.Zero(t, snapshot) -} - -func TestMessageMap_SubscribeSubset(t *testing.T) { - typedTestMessageMap_SubscribeSubset[*manager.AgentInfo](t, agentInfoCtor, agentInfoCmp) -} - -func typedTestMessageMap_SubscribeSubset[V watchable.Message](t *testing.T, ctor func(string) V, comp func(V, string) bool) { - ctx := dlog.NewTestContext(t, true) - var m watchable.Map[V] - - m.Store("a", ctor("A")) - m.Store("b", ctor("B")) - m.Store("c", ctor("C")) - - ch := m.SubscribeSubset(ctx, func(k string, v V) bool { - return !comp(v, "ignoreme") - }) - - // Check that a complete snapshot is immediately available - snapshot, ok := <-ch - assert.True(t, ok) - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{ - "a": ctor("A"), - "b": ctor("B"), - "c": ctor("C"), - }, - }, - snapshot) - - // Check that a no-op write doesn't trigger snapshot - m.Store("a", ctor("A")) - select { - case <-ch: - case <-time.After(10 * time.Millisecond): // just long enough that we have confidence <-ch isn't going to happen - } - - // Check that an overwrite triggers a new snapshot - m.Store("a", ctor("a")) - snapshot, ok = <-ch - assert.True(t, ok) - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{ - "a": ctor("a"), - "b": ctor("B"), - "c": ctor("C"), - }, - Updates: []watchable.Update[V]{ - {Key: "a", Value: ctor("a")}, - }, - }, - snapshot) - - // Check that a now-ignored entry gets deleted from the snapshot - m.Store("a", ctor("ignoreme")) - snapshot, ok = <-ch - assert.True(t, ok) - assertMessageMapSnapshotEqual(t, - watchable.Snapshot[V]{ - State: map[string]V{ - "b": ctor("B"), - "c": ctor("C"), - }, - Updates: []watchable.Update[V]{ - {Key: "a", Delete: true, Value: ctor("a")}, - }, - }, - snapshot) - - // Close the channel. For sake of test coverage, let's do some things different than in the - // non-Subset Subscribe test: - // 1. Use m.Close() to close *all* channels, rather than canceling the Context to close - // just the one (not that more than one exists in this test) - // 2. Don't have updates that will get coalesced in to the close. - m.Close() - snapshot, ok = <-ch - assert.False(t, ok) - assert.Zero(t, snapshot) - - // Now, since we've called m.Close(), let's check that subscriptions get already-closed - // channels. - ch = m.SubscribeSubset(ctx, func(k string, v V) bool { - return !comp(v, "ignoreme") - }) - snapshot, ok = <-ch - assert.False(t, ok) - assert.Zero(t, snapshot) -} diff --git a/cmd/traffic/cmd/manager/watchable/util_test.go b/cmd/traffic/cmd/manager/watchable/util_test.go deleted file mode 100644 index d94f62ce5c..0000000000 --- a/cmd/traffic/cmd/manager/watchable/util_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package watchable_test - -import ( - "fmt" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/proto" -) - -func assertDeepCopies(t *testing.T, a, b proto.Message, msgAndArgs ...any) bool { - t.Helper() - if a == b { - return assert.Fail(t, - fmt.Sprintf("Objects are pointer equal (are not copies):\n"+ - "a: %p\n"+ - "b: %p\n", - a, b), - msgAndArgs...) - } - if diff := cmp.Diff(a, b, cmp.Comparer(proto.Equal)); diff != "" { - return assert.Fail(t, "Not equal (-expected +actual):\n"+diff, msgAndArgs...) - } - return true -} diff --git a/docs/release-notes.md b/docs/release-notes.md index e96ecda404..a62d931e47 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -72,6 +72,18 @@ namespaceSelector: The output of the `telepresence list` command will now include the workload kind (deployment, replicaset, statefulset, or rollout) in all entries.
+##
change
Trigger the mutating webhook with Kubernetes eviction objects instead of patching workloads.
+
+ +Instead of patching workloads, or scaling the workloads down to zero and up again, Telepresence will now create policy/v1 Eviction objects to trigger the mutating webhook. This causes a slight change in the traffic-manager RBAC. The `patch` permissions are no longer needed. Instead, the traffic-manager must be able to create "pod/eviction" objects. +
+ +##
change
The telepresence-agents configmap is no longer used.
+
+ +The traffic-agent configuration was moved into a pod-annotation. This avoids sync problems between the telepresence-agents (which is no no longer present) and the pods. +
+ ##
change
Drop deprecated current-cluster-id command.
diff --git a/docs/release-notes.mdx b/docs/release-notes.mdx index 6fe709ff5c..6bcd43ede5 100644 --- a/docs/release-notes.mdx +++ b/docs/release-notes.mdx @@ -68,6 +68,14 @@ namespaceSelector: List output includes workload kind. The output of the `telepresence list` command will now include the workload kind (deployment, replicaset, statefulset, or rollout) in all entries. + + Trigger the mutating webhook with Kubernetes eviction objects instead of patching workloads. + Instead of patching workloads, or scaling the workloads down to zero and up again, Telepresence will now create policy/v1 Eviction objects to trigger the mutating webhook. This causes a slight change in the traffic-manager RBAC. The `patch` permissions are no longer needed. Instead, the traffic-manager must be able to create "pod/eviction" objects. + + + The telepresence-agents configmap is no longer used. + The traffic-agent configuration was moved into a pod-annotation. This avoids sync problems between the telepresence-agents (which is no no longer present) and the pods. + Drop deprecated current-cluster-id command. The clusterID was deprecated some time ago, and replaced by the ID of the namespace where the traffic-manager is installed. diff --git a/integration_test/injector_test.go b/integration_test/injector_test.go index df1a05f314..865dba4025 100644 --- a/integration_test/injector_test.go +++ b/integration_test/injector_test.go @@ -11,7 +11,6 @@ import ( "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/integration_test/itest" - "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" ) // Test_InterceptOperationRestoredAfterFailingInject tests that the telepresence-agents @@ -50,29 +49,10 @@ func (s *singleServiceSuite) Test_InterceptOperationRestoredAfterFailingInject() } }() - // Create an intercept again. This must succeed because nothing has changed - stdout = itest.TelepresenceOk(ctx, "intercept", s.ServiceName(), "--mount=false") - rq.Contains(stdout, "Using Deployment "+s.ServiceName()) - rq.Eventually(func() bool { - stdout, _, err := itest.Telepresence(ctx, "list", "--intercepts") - return err == nil && regexp.MustCompile(s.ServiceName()+`\s*: intercepted`).MatchString(stdout) - }, 12*time.Second, 3*time.Second) - itest.TelepresenceOk(ctx, "leave", s.ServiceName()) - // Uninstall the agent. This will remove it from the telepresence-agents configmap. It must also // uninstall from the agent, even though the webhook is muted, because there will be a rollout and // without the webhook, the default is that the pod has no agent. - func() { - // TODO: Uninstall should be CLI only and not require a traffic-manager connection - defer func() { - // Restore original user - itest.TelepresenceDisconnectOk(ctx) - s.TelepresenceConnect(ctx) - }() - itest.TelepresenceDisconnectOk(ctx) - s.TelepresenceConnect(itest.WithUser(ctx, "default")) - itest.TelepresenceOk(ctx, "uninstall", s.ServiceName()) - }() + itest.TelepresenceOk(ctx, "uninstall", s.ServiceName()) oneContainer := func() bool { pods := itest.RunningPodNames(ctx, s.ServiceName(), s.AppNamespace()) @@ -99,61 +79,16 @@ func (s *singleServiceSuite) Test_InterceptOperationRestoredAfterFailingInject() return false } - // Verify that the pod have no agent - rq.Eventually(oneContainer, 30*time.Second, 3*time.Second) - - // Now try to intercept. This will make the traffic-manager first inject an entry into the telepresence-agents - // configmap and the configmap watcher will then cause a subsequent rollout of the workload. That rollout - // would normally cause the mutating-webhook to kick in. That'll fail now because we disabled it further up. - iceptDone := make(chan error) - go func() { - defer close(iceptDone) - _, _, err := itest.Telepresence(ctx, "intercept", s.ServiceName(), "--mount=false") - select { - case <-ctx.Done(): - case iceptDone <- err: - } - }() - - const cmName = "telepresence-agents" - // Verify that there's a valid entry in the configmap - rq.Eventually(func() bool { - cmJSON, err := s.KubectlOut(ctx, "get", "configmap", cmName, "--output", "json") - if err != nil { - dlog.Errorf(ctx, "unable to get %s configmap: %v", cmName, err) - return false - } - var cm core.ConfigMap - err = json.Unmarshal([]byte(cmJSON), &cm) - if err != nil { - dlog.Errorf(ctx, "unable to parse json of %s configmap: %v", cmName, err) - return false - } - svcYAML, ok := cm.Data[s.ServiceName()] - if !ok { - dlog.Errorf(ctx, "didn't find an entry for %s in %s : %v", s.ServiceName(), cmName, err) - return false - } - sc, err := agentconfig.UnmarshalYAML([]byte(svcYAML)) - if err != nil { - dlog.Errorf(ctx, "unable to parse yaml of %s in the %s configmap: %v", s.ServiceName(), cmName, err) - } - return s.ServiceName() == sc.AgentConfig().AgentName - }, 30*time.Second, 3*time.Second) - - // Verify that the pod still have no agent + // Verify that the pod has no agent rq.Eventually(oneContainer, 30*time.Second, 3*time.Second) + // Now try to intercept. This attempt will timeout because the agent is never injected. + _, _, err := itest.Telepresence(ctx, "intercept", s.ServiceName(), "--mount=false") // Wait for the intercept call to return. It must return an error. - rq.Error(<-iceptDone) + rq.Error(err) - // Verify that the entry in the configmap has been removed by the traffic-manager. - cmJSON, err := s.KubectlOut(ctx, "get", "configmap", "telepresence-agents", "--output", "json") - rq.NoError(err) - var cm core.ConfigMap - rq.NoError(json.Unmarshal([]byte(cmJSON), &cm)) - _, ok := cm.Data[s.ServiceName()] - rq.False(ok) + // Verify that the pod still has no agent + rq.True(oneContainer()) // Restore mutating-webhook operation. rq.NoError(itest.Kubectl(ctx, s.ManagerNamespace(), "patch", "mutatingwebhookconfiguration", wh, diff --git a/integration_test/manual_agent_test.go b/integration_test/manual_agent_test.go index 48bc019fba..1c9ebf4d3b 100644 --- a/integration_test/manual_agent_test.go +++ b/integration_test/manual_agent_test.go @@ -8,11 +8,8 @@ import ( "strings" "time" - core "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" - "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/mutator" "github.com/telepresenceio/telepresence/v2/integration_test/itest" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" ) @@ -67,6 +64,10 @@ func testManualAgent(s *itest.Suite, nsp itest.NamespacePair) { var volumes []map[string]any require.NoError(yaml.Unmarshal([]byte(stdout), &volumes)) + stdout = itest.TelepresenceOk(ctx, "genyaml", "annotations", "--config", configFile) + var anns map[string]string + require.NoError(yaml.Unmarshal([]byte(stdout), &anns)) + b, err := os.ReadFile(filepath.Join(k8sDir, "echo-manual-inject-deploy.yaml")) require.NoError(err) var deploy map[string]any @@ -92,45 +93,7 @@ func testManualAgent(s *itest.Suite, nsp itest.NamespacePair) { podSpec["containers"] = append(cons, container) podSpec["initContainers"] = []map[string]any{initContainer} podSpec["volumes"] = volumes - podTemplate["metadata"].(map[string]any)["annotations"] = map[string]string{mutator.ManualInjectAnnotation: "true"} - - // Add the configmap entry by first retrieving the current config map - var cfgMap *core.ConfigMap - origCfgYaml, err := nsp.KubectlOut(ctx, "get", "configmap", agentconfig.ConfigMap, "-o", "yaml") - if err != nil { - cfgMap = &core.ConfigMap{ - TypeMeta: meta.TypeMeta{ - Kind: "ConfigMap", - APIVersion: "v1", - }, - ObjectMeta: meta.ObjectMeta{ - Name: agentconfig.ConfigMap, - Namespace: nsp.AppNamespace(), - }, - } - origCfgYaml = "" - } else { - require.NoError(yaml.Unmarshal([]byte(origCfgYaml), &cfgMap)) - } - if cfgMap.Data == nil { - cfgMap.Data = make(map[string]string) - } - cfgMap.Data[ac.WorkloadName] = cfgEntry - - cfgYaml := writeYaml(agentconfig.ConfigMap+".yaml", cfgMap) - require.NoError(nsp.Kubectl(ctx, "apply", "-f", cfgYaml)) - defer func() { - if origCfgYaml == "" { - require.NoError(nsp.Kubectl(ctx, "delete", "configmap", agentconfig.ConfigMap)) - } else { - // Restore original configmap - cfgMap.ObjectMeta = meta.ObjectMeta{ - Name: agentconfig.ConfigMap, - Namespace: nsp.AppNamespace(), - } - writeYaml(agentconfig.ConfigMap+".yaml", cfgMap) - } - }() + podTemplate["metadata"].(map[string]any)["annotations"] = anns dplYaml := writeYaml("deployment.yaml", deploy) require.NoError(nsp.Kubectl(ctx, "apply", "-f", dplYaml)) diff --git a/integration_test/multiport_test.go b/integration_test/multiport_test.go index 67a6518dd3..0ed71be920 100644 --- a/integration_test/multiport_test.go +++ b/integration_test/multiport_test.go @@ -192,7 +192,9 @@ func (s *connectedSuite) Test_SameContainerPort() { ctx := s.Context() dep := "echo-stp" s.ApplyApp(ctx, "echo-same-target-port", "deploy/"+dep) - defer s.KubectlOk(ctx, "delete", "deploy", dep) + defer func() { + s.KubectlOk(ctx, "delete", "deploy", dep) + }() portTest := func(svcPort string) { ctx := s.Context() diff --git a/integration_test/workload_configuration_test.go b/integration_test/workload_configuration_test.go index b7ccd2be1b..13c9cfe2f5 100644 --- a/integration_test/workload_configuration_test.go +++ b/integration_test/workload_configuration_test.go @@ -1,7 +1,6 @@ package integration_test import ( - "context" "fmt" "strings" "time" @@ -31,8 +30,6 @@ func (s *workloadConfigurationSuite) disabledWorkloadKind(tp, wl string) { s.ApplyApp(ctx, wl, strings.ToLower(tp)+"/"+wl) defer s.DeleteSvcAndWorkload(ctx, strings.ToLower(tp), wl) - defer s.uninstallAgents(ctx, wl) - s.TelepresenceConnect(ctx) defer itest.TelepresenceDisconnectOk(ctx) @@ -47,13 +44,6 @@ func (s *workloadConfigurationSuite) disabledWorkloadKind(tp, wl string) { require.Contains(stderr, fmt.Sprintf("connector.CreateIntercept: workload \"%s.%s\" not found", wl, s.AppNamespace())) } -func (s *workloadConfigurationSuite) uninstallAgents(ctx context.Context, wl string) { - dfltCtx := itest.WithUser(ctx, "default") - itest.TelepresenceOk(dfltCtx, "connect", "--namespace", s.AppNamespace(), "--manager-namespace", s.ManagerNamespace()) - itest.TelepresenceOk(dfltCtx, "uninstall", wl) - itest.TelepresenceDisconnectOk(dfltCtx) -} - func (s *workloadConfigurationSuite) Test_DisabledReplicaSet() { s.TelepresenceHelmInstallOK(s.Context(), true, "--set", "workloads.replicaSets.enabled=false") defer s.TelepresenceHelmInstallOK(s.Context(), true, "--set", "workloads.replicaSets.enabled=true") @@ -77,8 +67,6 @@ func (s *workloadConfigurationSuite) Test_InterceptsDeploymentWithDisabledReplic s.TelepresenceHelmInstallOK(ctx, true, "--set", "workloads.replicaSets.enabled=false") defer s.TelepresenceHelmInstallOK(ctx, true, "--set", "workloads.replicaSets.enabled=true") - defer s.uninstallAgents(ctx, wl) - s.TelepresenceConnect(ctx) defer itest.TelepresenceDisconnectOk(ctx) @@ -112,8 +100,6 @@ func (s *workloadConfigurationSuite) Test_InterceptsReplicaSetWithDisabledDeploy s.TelepresenceHelmInstallOK(ctx, true, "--set", "workloads.deployments.enabled=false") defer s.TelepresenceHelmInstallOK(ctx, true, "--set", "workloads.deployments.enabled=true") - defer s.uninstallAgents(ctx, interceptableWl) - s.TelepresenceConnect(ctx) defer itest.TelepresenceDisconnectOk(ctx) diff --git a/pkg/agentconfig/container.go b/pkg/agentconfig/container.go index 1820e7d99e..4aa99ed10c 100644 --- a/pkg/agentconfig/container.go +++ b/pkg/agentconfig/container.go @@ -2,6 +2,7 @@ package agentconfig import ( "context" + "fmt" "regexp" "strconv" "strings" @@ -24,10 +25,10 @@ func AgentContainer( confCns := ConfiguredContainers(ctx, pod, config) eachConfiguredContainer(confCns, config, func(app *core.Container, cc *Container) { - if cc.Replace { + if cc.Replace == ReplacePolicyContainer { // Simply inherit the ports of the replaced container ports = append(ports, app.Ports...) - } else { + } else if cc.Replace == ReplacePolicyIntercept { for _, ic := range PortUniqueIntercepts(cc) { ports = append(ports, core.ContainerPort{ Name: ic.ContainerPortName, @@ -51,6 +52,15 @@ func AgentContainer( }) } evs = append(evs, + core.EnvVar{ + Name: "AGENT_CONFIG", + ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: fmt.Sprintf("metadata.annotations['%s']", ConfigAnnotation), + }, + }, + }, core.EnvVar{ Name: EnvPrefixAgent + "POD_IP", ValueFrom: &core.EnvVarSource{ @@ -94,10 +104,6 @@ func AgentContainer( Name: AnnotationVolumeName, MountPath: AnnotationMountPoint, }, - core.VolumeMount{ - Name: ConfigVolumeName, - MountPath: ConfigMountPoint, - }, core.VolumeMount{ Name: ExportsVolumeName, MountPath: ExportsMountPoint, @@ -137,17 +143,19 @@ func AgentContainer( efs = nil } - replaceAnnotations := make(map[string]string) + annotations := make(map[string]string) eachConfiguredContainer(confCns, config, func(app *core.Container, cc *Container) { - if cc.Replace { + if cc.Replace == ReplacePolicyContainer { cnJson, err := json.Marshal(app) if err != nil { dlog.Errorf(ctx, "unable to marshal container %s.%s/%s to json: %v", config.WorkloadName, config.Namespace, app.Name, err) } else { - replaceAnnotations[ReplacedContainerAnnotationPrefix+cc.Name] = string(cnJson) + annotations[ReplaceAnnotationKey(cc.Name)] = string(cnJson) } } }) + cfg, _ := MarshalTight(config) + annotations[ConfigAnnotation] = cfg if len(ports) == 0 { ports = nil @@ -185,7 +193,11 @@ func AgentContainer( } ac.SecurityContext = appSc - return ac, replaceAnnotations + return ac, annotations +} + +func ReplaceAnnotationKey(cn string) string { + return ReplacedContainerAnnotationPrefix + cn } // Find the security context of the first container (with both intercepts and a set security context) and ensure @@ -228,6 +240,15 @@ func InitContainer(config *Sidecar) *core.Container { Name: "LOG_LEVEL", Value: config.LogLevel, }, + { + Name: "AGENT_CONFIG", + ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: fmt.Sprintf("metadata.annotations['%s']", ConfigAnnotation), + }, + }, + }, { Name: "POD_IP", ValueFrom: &core.EnvVarSource{ @@ -238,10 +259,6 @@ func InitContainer(config *Sidecar) *core.Container { }, }, }, - VolumeMounts: []core.VolumeMount{{ - Name: ConfigVolumeName, - MountPath: ConfigMountPoint, - }}, SecurityContext: &core.SecurityContext{ Capabilities: &core.Capabilities{ Add: []core.Capability{"NET_ADMIN"}, @@ -255,13 +272,6 @@ func InitContainer(config *Sidecar) *core.Container { } func AgentVolumes(agentName string, pod *core.Pod) []core.Volume { - var items []core.KeyToPath - if agentName != "" { - items = []core.KeyToPath{{ - Key: agentName, - Path: ConfigFile, - }} - } volumes := []core.Volume{ { Name: AnnotationVolumeName, @@ -279,15 +289,6 @@ func AgentVolumes(agentName string, pod *core.Pod) []core.Volume { }, }, }, - { - Name: ConfigVolumeName, - VolumeSource: core.VolumeSource{ - ConfigMap: &core.ConfigMapVolumeSource{ - LocalObjectReference: core.LocalObjectReference{Name: ConfigMap}, - Items: items, - }, - }, - }, { Name: ExportsVolumeName, VolumeSource: core.VolumeSource{ diff --git a/pkg/agentconfig/container_test.go b/pkg/agentconfig/container_test.go index 39d2a60ff3..a39d5cdade 100644 --- a/pkg/agentconfig/container_test.go +++ b/pkg/agentconfig/container_test.go @@ -2,10 +2,6 @@ package agentconfig import ( "testing" - - "github.com/go-json-experiment/json" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func Test_prefixInterpolated(t *testing.T) { @@ -73,28 +69,3 @@ func Test_prefixInterpolated(t *testing.T) { }) } } - -func Test_ReplacePolicy(t *testing.T) { - var cn Container - require.NoError(t, json.Unmarshal([]byte(`{"replace": 0}`), &cn)) - assert.False(t, bool(cn.Replace)) - require.NoError(t, json.Unmarshal([]byte(`{}`), &cn)) - assert.False(t, bool(cn.Replace)) - require.NoError(t, json.Unmarshal([]byte(`{"replace":1}`), &cn)) - assert.True(t, bool(cn.Replace)) - require.NoError(t, json.Unmarshal([]byte(`{"replace":2}`), &cn)) - assert.False(t, bool(cn.Replace)) - require.NoError(t, json.Unmarshal([]byte(`{"replace":false}`), &cn)) - assert.False(t, bool(cn.Replace)) - require.NoError(t, json.Unmarshal([]byte(`{"replace":true}`), &cn)) - assert.True(t, bool(cn.Replace)) - - cn.Replace = false - data, err := json.Marshal(&cn) - require.NoError(t, err) - require.Equal(t, string(data), `{}`) - cn.Replace = true - data, err = json.Marshal(&cn) - require.NoError(t, err) - require.Equal(t, string(data), `{"replace":1}`) -} diff --git a/pkg/agentconfig/sidecar.go b/pkg/agentconfig/sidecar.go index e09fbd4292..6e2db2fa5a 100644 --- a/pkg/agentconfig/sidecar.go +++ b/pkg/agentconfig/sidecar.go @@ -23,7 +23,6 @@ const ( TerminatingTLSMountPoint = "/terminating_tls" OriginatingTLSVolumeName = "traffic-originating-tls" OriginatingTLSMountPoint = "/originating_tls" - ConfigFile = "config.yaml" MountPrefixApp = "/tel_app_mounts" ExportsVolumeName = "export-volume" ExportsMountPoint = "/tel_app_exports" @@ -33,6 +32,9 @@ const ( EnvPrefixAgent = EnvPrefix + "AGENT_" EnvPrefixApp = EnvPrefix + "APP_" + // EnvAgentConfig is the environment variable where the traffic-agent finds its own config. + EnvAgentConfig = "AGENT_CONFIG" + // EnvInterceptContainer intercepted container propagated to client during intercept. EnvInterceptContainer = "TELEPRESENCE_CONTAINER" @@ -42,11 +44,15 @@ const ( // EnvAPIPort is the port number of the Telepresence API server, when it is enabled. EnvAPIPort = "TELEPRESENCE_API_PORT" - DomainPrefix = "telepresence.getambassador.io/" + DomainPrefix = "telepresence.getambassador.io/" + + RestartedAtAnnotation = DomainPrefix + "restartedAt" + ManualInjectAnnotation = DomainPrefix + "manually-injected" InjectAnnotation = DomainPrefix + "inject-" + ContainerName InjectIgnoreVolumeMounts = DomainPrefix + "inject-ignore-volume-mounts" TerminatingTLSSecretAnnotation = DomainPrefix + "inject-terminating-tls-secret" OriginatingTLSSecretAnnotation = DomainPrefix + "inject-originating-tls-secret" + ConfigAnnotation = DomainPrefix + "agent-config" ReplacedContainerAnnotationPrefix = DomainPrefix + "replaced-container." LegacyTerminatingTLSSecretAnnotation = "getambassador.io/inject-terminating-tls-secret" LegacyOriginatingTLSSecretAnnotation = "getambassador.io/inject-originating-tls-secret" @@ -56,30 +62,22 @@ const ( K8SCreatedByLabel = "app.kubernetes.io/created-by" ) -type ReplacePolicy bool - -func (r *ReplacePolicy) UnmarshalJSON(data []byte) error { - var i int - if err := json.Unmarshal(data, &i); err != nil { - // Allow true/false too. - var v bool - if boolErr := json.Unmarshal(data, &v); boolErr != nil { - return err - } - *r = ReplacePolicy(v) - } else { - *r = i == 1 - } - return nil -} +type ReplacePolicy int -func (r ReplacePolicy) MarshalJSON() ([]byte, error) { - i := 0 - if r { - i = 1 - } - return json.Marshal(&i) -} +const ( + // ReplacePolicyIntercept The traffic-agent will receive all traffic intended for the ports of the app-container and + // then either route that traffic to the client or to the original app-container depending on if the port is + // intercepted or not. This will require an init-container when the targetPort of the service is numeric or + // when the service is headless. + ReplacePolicyIntercept ReplacePolicy = iota + + // ReplacePolicyContainer The traffic-agent is currently replacing the app container and routes all traffic to the + // client. + ReplacePolicyContainer + + // ReplacePolicyInactive The traffic-agent is not interfering with any ports or containers. + ReplacePolicyInactive +) // Intercept describes the mapping between a service port and an intercepted container port or, when // service is used, just the container port. @@ -135,7 +133,7 @@ type Container struct { // Mounts are the actual mount points that are mounted by this container Mounts []string `json:"Mounts,omitempty"` - // Replace is whether the agent should replace the intercepted container + // Replace is whether the agent should replace the intercepted container, it's ports, or nothing. Replace ReplacePolicy `json:"replace,omitzero"` } @@ -197,6 +195,20 @@ func (s *Sidecar) AgentConfig() *Sidecar { return s } +// Clone returns a deep copy of the SidecarExt. +func (s *Sidecar) Clone() SidecarExt { + cs := *s + for ci, cn := range cs.Containers { + ccn := *cn + cs.Containers[ci] = &ccn + for ii, ic := range ccn.Intercepts { + cic := *ic + ccn.Intercepts[ii] = &cic + } + } + return &cs +} + // Marshal returns YAML encoding of the Sidecar. func (s *Sidecar) Marshal() ([]byte, error) { return yaml.Marshal(s) @@ -208,6 +220,8 @@ type SidecarExt interface { AgentConfig() *Sidecar Marshal() ([]byte, error) + + Clone() SidecarExt } // SidecarType is Sidecar by default but can be any type implementing SidecarExt. @@ -221,3 +235,43 @@ func UnmarshalYAML(data []byte) (SidecarExt, error) { } return into.(SidecarExt), nil } + +// MarshalTight marshals the given instance into JSON data, with data relating to the creation of the +// container manifest stripped off. +func MarshalTight(s SidecarExt) (string, error) { + ac := s.AgentConfig() + + // Strip things that are not needed once the container has been created. + ai := ac.AgentImage + pp := ac.PullPolicy + ps := ac.PullSecrets + ir := ac.InitResources + sc := ac.SecurityContext + + ac.AgentImage = "" + ac.PullPolicy = "" + ac.PullSecrets = nil + ac.InitResources = nil + ac.SecurityContext = nil + + data, err := json.Marshal(s) + ac.AgentImage = ai + ac.PullPolicy = pp + ac.PullSecrets = ps + ac.InitResources = ir + ac.SecurityContext = sc + + if err != nil { + return "", err + } + return string(data), err +} + +// UnmarshalJSON creates a new instance of the SidecarType from the given JSON data. +func UnmarshalJSON(data string) (SidecarExt, error) { + into := reflect.New(SidecarType).Interface() + if err := json.Unmarshal([]byte(data), into); err != nil { + return nil, err + } + return into.(SidecarExt), nil +} diff --git a/pkg/client/cli/cmd/genyaml.go b/pkg/client/cli/cmd/genyaml.go index 614ec6fb1f..37c6dde1be 100644 --- a/pkg/client/cli/cmd/genyaml.go +++ b/pkg/client/cli/cmd/genyaml.go @@ -44,15 +44,12 @@ func genYAML() *cobra.Command { Short: "Generate YAML for use in kubernetes manifests.", Long: `Generate traffic-agent yaml for use in kubernetes manifests. This allows the traffic agent to be injected by hand into existing kubernetes manifests. -For your modified workload to be valid, you'll have to manually inject a container and a -volume into the workload, and a corresponding configmap entry into the "telelepresence-agents" -configmap; you can do this by running "genyaml config", "genyaml container", and "genyaml volume". +For your modified workload to be valid, you'll have to manually inject annotations, a +container, and a volume into the workload; you can do this by running "genyaml config", +"genyaml container", "genyaml initcontainer", "genyaml annotations", and "genyaml volume". NOTE: It is recommended that you not do this unless strictly necessary. Instead, we suggest letting telepresence's webhook injector configure the traffic agents on demand.`, - RunE: func(_ *cobra.Command, _ []string) error { - return errcat.User.New("please run genyaml as \"genyaml config\", \"genyaml container\", \"genyaml initcontainer\", or \"genyaml volume\"") - }, ValidArgsFunction: cobra.NoFileCompletions, } flags := cmd.PersistentFlags() @@ -62,6 +59,7 @@ telepresence's webhook injector configure the traffic agents on demand.`, genConfigMapSubCommand(&info), genContainerSubCommand(&info), genInitContainerSubCommand(&info), + genVAnnotationsSubCommand(&info), genVolumeSubCommand(&info), ) return cmd @@ -421,6 +419,55 @@ func (g *genInitContainerInfo) run(cmd *cobra.Command, kubeFlags map[string]stri return errcat.User.New("deployment does not need an init container") } +type genAnnotationsInfo struct { + *genYAMLCommand +} + +func genVAnnotationsSubCommand(yamlInfo *genYAMLCommand) *cobra.Command { + info := genAnnotationsInfo{genYAMLCommand: yamlInfo} + kubeFlags := allKubeFlags() + cmd := &cobra.Command{ + Use: "annotations", + Args: cobra.NoArgs, + Short: "Generate YAML for the pod template metadata annotations.", + Long: "Generate YAML for the pod template metadata annotations. See genyaml for more info on what this means", + RunE: func(cmd *cobra.Command, args []string) error { + return info.run(cmd, flags.Map(kubeFlags)) + }, + } + flags := cmd.Flags() + flags.StringVarP(&info.workloadName, "workload", "w", "", + "Name of the workload. If given, the configmap entry will be retrieved telepresence-agents configmap, mutually exclusive to --config") + flags.StringVarP(&info.configFile, "config", "c", "", "Path to the yaml containing the generated configmap entry, mutually exclusive to --workload") + flags.AddFlagSet(kubeFlags) + return cmd +} + +func (g *genAnnotationsInfo) run(cmd *cobra.Command, kubeFlags map[string]string) error { + ctx := cmd.Context() + if g.configFile == "" { + var err error + ctx, err = g.WithJoinedClientSetInterface(ctx, kubeFlags) + if err != nil { + return err + } + } + cm, err := g.loadConfigMapEntry(ctx) + if err != nil { + return err + } + cmJSON, err := agentconfig.MarshalTight(cm) + if err != nil { + return err + } + annotations := map[string]string{ + agentconfig.InjectAnnotation: "enabled", + agentconfig.ManualInjectAnnotation: "true", + agentconfig.ConfigAnnotation: cmJSON, + } + return g.writeObjToOutput(annotations) +} + type genVolumeInfo struct { *genYAMLCommand } diff --git a/pkg/client/userd/trafficmgr/session.go b/pkg/client/userd/trafficmgr/session.go index a3559c4a99..ec7a374411 100644 --- a/pkg/client/userd/trafficmgr/session.go +++ b/pkg/client/userd/trafficmgr/session.go @@ -900,6 +900,19 @@ func (s *session) status(c context.Context, initial bool) *rpc.ConnectInfo { // // Uninstalling all or specific agents require that the client can get and update the agents ConfigMap. func (s *session) Uninstall(ctx context.Context, ur *rpc.UninstallRequest) (*common.Result, error) { + _, err := s.managerClient.UninstallAgents(ctx, &manager.UninstallAgentsRequest{ + SessionInfo: s.sessionInfo, + Agents: ur.Agents, + }) + if err != nil { + if status.Code(err) == codes.Unimplemented { + return s.legacyUninstall(ctx, ur) + } + } + return errcat.ToResult(err), nil +} + +func (s *session) legacyUninstall(ctx context.Context, ur *rpc.UninstallRequest) (*common.Result, error) { api := k8sapi.GetK8sInterface(ctx).CoreV1() loadAgentConfigMap := func(ns string) (*core.ConfigMap, error) { cm, err := api.ConfigMaps(ns).Get(ctx, agentconfig.ConfigMap, meta.GetOptions{}) @@ -1250,10 +1263,11 @@ func (s *session) workloadsWatcher(ctx context.Context, namespace string, synced clients[i] = ic.Client } } - dlog.Debugf(ctx, "Adding workload %s/%s.%s", key.kind, key.name, namespace) + state := workload.StateFromRPC(w.State) + dlog.Debugf(ctx, "Adding workload %s/%s.%s %s %s %s", key.kind, key.name, namespace, state, w.AgentState, clients) workloads[key] = workloadInfo{ uid: types.UID(w.Uid), - state: workload.StateFromRPC(w.State), + state: state, agentState: w.AgentState, interceptClients: clients, } diff --git a/pkg/workload/util.go b/pkg/workload/util.go index 429497ba89..21f2a7a27c 100644 --- a/pkg/workload/util.go +++ b/pkg/workload/util.go @@ -3,18 +3,9 @@ package workload import ( "k8s.io/apimachinery/pkg/runtime" - "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" ) -const ( - DomainPrefix = "telepresence.getambassador.io/" - InjectAnnotation = DomainPrefix + "inject-" + agentconfig.ContainerName - ServiceNameAnnotation = DomainPrefix + "inject-service-name" - ManualInjectAnnotation = DomainPrefix + "manually-injected" - AnnRestartedAt = DomainPrefix + "restartedAt" -) - func FromAny(obj any) (k8sapi.Workload, bool) { if ro, ok := obj.(runtime.Object); ok { if wl, err := k8sapi.WrapWorkload(ro); err == nil { diff --git a/pkg/workload/watcher.go b/pkg/workload/watcher.go index 50e47c3d28..3eefbd8b88 100644 --- a/pkg/workload/watcher.go +++ b/pkg/workload/watcher.go @@ -17,6 +17,7 @@ import ( "k8s.io/kubectl/pkg/util/deployment" "github.com/datawire/dlib/dlog" + "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/informer" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" @@ -214,7 +215,7 @@ func compareOptions() []cmp.Option { // Ignore frequently changing annotations of no interest. cmpopts.IgnoreMapEntries(func(k, _ string) bool { - return k == AnnRestartedAt || k == deployment.RevisionAnnotation + return k == agentconfig.RestartedAtAnnotation || k == deployment.RevisionAnnotation }), } } diff --git a/rpc/manager/manager.pb.go b/rpc/manager/manager.pb.go index cefa2ab0e2..b34d6d90c0 100644 --- a/rpc/manager/manager.pb.go +++ b/rpc/manager/manager.pb.go @@ -3857,6 +3857,61 @@ func (x *WorkloadEventsRequest) GetNamespace() string { return "" } +type UninstallAgentsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The session_info identifies the client connection, and hence the + // namespace for the resulting watcher. + SessionInfo *SessionInfo `protobuf:"bytes,1,opt,name=session_info,json=sessionInfo,proto3" json:"session_info,omitempty"` + // The agents to install. Empty means all agents in the connected namespace. + Agents []string `protobuf:"bytes,2,rep,name=agents,proto3" json:"agents,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UninstallAgentsRequest) Reset() { + *x = UninstallAgentsRequest{} + mi := &file_manager_manager_proto_msgTypes[48] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UninstallAgentsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UninstallAgentsRequest) ProtoMessage() {} + +func (x *UninstallAgentsRequest) ProtoReflect() protoreflect.Message { + mi := &file_manager_manager_proto_msgTypes[48] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UninstallAgentsRequest.ProtoReflect.Descriptor instead. +func (*UninstallAgentsRequest) Descriptor() ([]byte, []int) { + return file_manager_manager_proto_rawDescGZIP(), []int{48} +} + +func (x *UninstallAgentsRequest) GetSessionInfo() *SessionInfo { + if x != nil { + return x.SessionInfo + } + return nil +} + +func (x *UninstallAgentsRequest) GetAgents() []string { + if x != nil { + return x.Agents + } + return nil +} + // "Mechanisms" are the ways that an Agent can decide handle // incoming requests, and decide whether to send them to the // in-cluster service, or whether to intercept them. The "tcp" @@ -3879,7 +3934,7 @@ type AgentInfo_Mechanism struct { func (x *AgentInfo_Mechanism) Reset() { *x = AgentInfo_Mechanism{} - mi := &file_manager_manager_proto_msgTypes[48] + mi := &file_manager_manager_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3891,7 +3946,7 @@ func (x *AgentInfo_Mechanism) String() string { func (*AgentInfo_Mechanism) ProtoMessage() {} func (x *AgentInfo_Mechanism) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[48] + mi := &file_manager_manager_proto_msgTypes[49] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3940,7 +3995,7 @@ type AgentInfo_ContainerInfo struct { func (x *AgentInfo_ContainerInfo) Reset() { *x = AgentInfo_ContainerInfo{} - mi := &file_manager_manager_proto_msgTypes[49] + mi := &file_manager_manager_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3952,7 +4007,7 @@ func (x *AgentInfo_ContainerInfo) String() string { func (*AgentInfo_ContainerInfo) ProtoMessage() {} func (x *AgentInfo_ContainerInfo) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[49] + mi := &file_manager_manager_proto_msgTypes[50] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3992,7 +4047,7 @@ type WorkloadInfo_Intercept struct { func (x *WorkloadInfo_Intercept) Reset() { *x = WorkloadInfo_Intercept{} - mi := &file_manager_manager_proto_msgTypes[61] + mi := &file_manager_manager_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4004,7 +4059,7 @@ func (x *WorkloadInfo_Intercept) String() string { func (*WorkloadInfo_Intercept) ProtoMessage() {} func (x *WorkloadInfo_Intercept) ProtoReflect() protoreflect.Message { - mi := &file_manager_manager_proto_msgTypes[61] + mi := &file_manager_manager_proto_msgTypes[62] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4691,218 +4746,231 @@ var file_manager_manager_proto_rawDesc = string([]byte{ 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x2a, 0xad, 0x01, - 0x0a, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, 0x6f, - 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, - 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x57, 0x41, 0x49, 0x54, 0x49, - 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, - 0x09, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x03, - 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x10, 0x04, 0x12, 0x10, - 0x0a, 0x0c, 0x4e, 0x4f, 0x5f, 0x4d, 0x45, 0x43, 0x48, 0x41, 0x4e, 0x49, 0x53, 0x4d, 0x10, 0x05, - 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x50, 0x4f, 0x52, 0x54, 0x53, 0x10, 0x06, 0x12, 0x0f, - 0x0a, 0x0b, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x07, 0x12, - 0x0c, 0x0a, 0x08, 0x42, 0x41, 0x44, 0x5f, 0x41, 0x52, 0x47, 0x53, 0x10, 0x08, 0x32, 0xbc, 0x18, - 0x0a, 0x07, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x45, 0x0a, 0x07, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x22, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x32, - 0x12, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, - 0x65, 0x46, 0x51, 0x4e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x23, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, - 0x4e, 0x12, 0x65, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4c, - 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, - 0x19, 0x43, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x6d, 0x62, 0x61, 0x73, - 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, - 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x2b, 0x2e, + 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x76, 0x0a, + 0x16, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x44, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, - 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4a, 0x0a, 0x0f, 0x47, 0x65, - 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x4c, 0x49, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x57, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x54, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x41, 0x50, 0x49, 0x12, 0x16, 0x2e, 0x67, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, + 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x61, + 0x67, 0x65, 0x6e, 0x74, 0x73, 0x2a, 0xad, 0x01, 0x0a, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, + 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x01, 0x12, + 0x0b, 0x0a, 0x07, 0x57, 0x41, 0x49, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, + 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, 0x09, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, + 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x41, + 0x47, 0x45, 0x4e, 0x54, 0x10, 0x04, 0x12, 0x10, 0x0a, 0x0c, 0x4e, 0x4f, 0x5f, 0x4d, 0x45, 0x43, + 0x48, 0x41, 0x4e, 0x49, 0x53, 0x4d, 0x10, 0x05, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x50, + 0x4f, 0x52, 0x54, 0x53, 0x10, 0x06, 0x12, 0x0f, 0x0a, 0x0b, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, + 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x07, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x41, 0x44, 0x5f, 0x41, + 0x52, 0x47, 0x53, 0x10, 0x08, 0x32, 0x95, 0x19, 0x0a, 0x07, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x12, 0x45, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x41, 0x50, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x55, 0x0a, 0x0e, 0x41, 0x72, 0x72, 0x69, 0x76, 0x65, 0x41, 0x73, 0x43, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x53, 0x0a, 0x0d, 0x41, 0x72, 0x72, 0x69, 0x76, 0x65, - 0x41, 0x73, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x45, 0x0a, 0x06, 0x52, - 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, - 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x12, 0x43, 0x0a, 0x06, 0x44, 0x65, 0x70, 0x61, 0x72, 0x74, 0x12, 0x21, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4c, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, - 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, - 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x65, 0x0a, 0x0e, 0x47, 0x65, 0x74, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, + 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x43, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x69, + 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x19, 0x43, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, + 0x75, 0x64, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x2f, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x2e, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, + 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x0e, 0x47, + 0x65, 0x74, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, - 0x12, 0x24, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, - 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x57, 0x61, - 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x73, 0x12, 0x21, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, - 0x2a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, - 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x5b, 0x0a, - 0x0b, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x21, 0x2e, 0x74, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6d, 0x62, + 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x12, 0x4a, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x4c, 0x49, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x57, + 0x0a, 0x12, 0x47, 0x65, 0x74, 0x54, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x41, 0x50, 0x49, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, - 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, - 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x5f, 0x0a, 0x0d, 0x57, 0x61, - 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x4e, 0x53, 0x12, 0x23, 0x2e, 0x74, 0x65, + 0x67, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x41, 0x50, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x55, 0x0a, 0x0e, 0x41, 0x72, 0x72, 0x69, 0x76, + 0x65, 0x41, 0x73, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, - 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x63, 0x0a, 0x0f, 0x57, - 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x73, 0x12, 0x21, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, - 0x6f, 0x1a, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, - 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, - 0x12, 0x6a, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, - 0x64, 0x73, 0x12, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, - 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x73, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x30, 0x01, 0x12, 0x5a, 0x0a, 0x10, - 0x57, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, - 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x53, + 0x0a, 0x0d, 0x41, 0x72, 0x72, 0x69, 0x76, 0x65, 0x41, 0x73, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, + 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, + 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, - 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, 0x12, 0x60, 0x0a, 0x0b, 0x45, 0x6e, 0x73, 0x75, - 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x45, - 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, - 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x69, 0x0a, 0x10, 0x50, 0x72, - 0x65, 0x70, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, + 0x6e, 0x66, 0x6f, 0x12, 0x45, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x23, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x06, 0x44, 0x65, + 0x70, 0x61, 0x72, 0x74, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, + 0x4c, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x64, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, + 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x24, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x58, 0x0a, 0x0f, 0x52, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2d, + 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0x1a, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, + 0x50, 0x6f, 0x64, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x2a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x5b, 0x0a, 0x0b, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, + 0x30, 0x01, 0x12, 0x5f, 0x0a, 0x0d, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, + 0x73, 0x4e, 0x53, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x5e, 0x0a, 0x0c, 0x47, - 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x29, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x57, 0x0a, 0x0f, 0x52, - 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, - 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x21, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, - 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, - 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x09, 0x4c, 0x6f, - 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, - 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, + 0x74, 0x30, 0x01, 0x12, 0x63, 0x0a, 0x0f, 0x57, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x63, 0x65, 0x70, 0x74, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x16, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, - 0x53, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, - 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, + 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x6a, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, + 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x12, 0x2b, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, + 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x44, 0x65, 0x6c, + 0x74, 0x61, 0x30, 0x01, 0x12, 0x5a, 0x0a, 0x10, 0x57, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x20, 0x2e, 0x74, 0x65, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x12, - 0x50, 0x0a, 0x0d, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, + 0x12, 0x60, 0x0a, 0x0b, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, + 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, + 0x6f, 0x74, 0x12, 0x69, 0x0a, 0x10, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x70, + 0x61, 0x72, 0x65, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x64, 0x0a, + 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x58, 0x0a, 0x0f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x32, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, + 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x5e, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, + 0x65, 0x70, 0x74, 0x12, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x57, 0x0a, 0x0f, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, + 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, 0x15, + 0x47, 0x65, 0x74, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, + 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, - 0x01, 0x12, 0x56, 0x0a, 0x06, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x23, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x70, - 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, 0x09, 0x57, 0x61, 0x74, 0x63, 0x68, - 0x44, 0x69, 0x61, 0x6c, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, - 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x42, 0x37, 0x5a, 0x35, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x69, 0x6f, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x32, 0x2f, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, + 0x64, 0x73, 0x12, 0x50, 0x0a, 0x09, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x12, + 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x16, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x6f, + 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, + 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, + 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x66, 0x6f, 0x1a, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x12, 0x50, 0x0a, 0x0d, 0x57, 0x61, 0x74, 0x63, 0x68, + 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x1a, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x12, 0x56, 0x0a, 0x06, 0x54, 0x75, 0x6e, + 0x6e, 0x65, 0x6c, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, + 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, + 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, + 0x01, 0x12, 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, + 0x53, 0x0a, 0x09, 0x57, 0x61, 0x74, 0x63, 0x68, 0x44, 0x69, 0x61, 0x6c, 0x12, 0x21, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, + 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x30, 0x01, 0x12, 0x57, 0x0a, 0x0f, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, + 0x6c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x55, + 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x37, 0x5a, + 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x69, 0x6f, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x32, 0x2f, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, }) var ( @@ -4918,7 +4986,7 @@ func file_manager_manager_proto_rawDescGZIP() []byte { } var file_manager_manager_proto_enumTypes = make([]protoimpl.EnumInfo, 5) -var file_manager_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 62) +var file_manager_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 63) var file_manager_manager_proto_goTypes = []any{ (InterceptDispositionType)(0), // 0: telepresence.manager.InterceptDispositionType (WorkloadInfo_Kind)(0), // 1: telepresence.manager.WorkloadInfo.Kind @@ -4973,37 +5041,38 @@ var file_manager_manager_proto_goTypes = []any{ (*WorkloadEvent)(nil), // 50: telepresence.manager.WorkloadEvent (*WorkloadEventsDelta)(nil), // 51: telepresence.manager.WorkloadEventsDelta (*WorkloadEventsRequest)(nil), // 52: telepresence.manager.WorkloadEventsRequest - (*AgentInfo_Mechanism)(nil), // 53: telepresence.manager.AgentInfo.Mechanism - (*AgentInfo_ContainerInfo)(nil), // 54: telepresence.manager.AgentInfo.ContainerInfo - nil, // 55: telepresence.manager.AgentInfo.ContainersEntry - nil, // 56: telepresence.manager.AgentInfo.ContainerInfo.EnvironmentEntry - nil, // 57: telepresence.manager.PreviewSpec.AddRequestHeadersEntry - nil, // 58: telepresence.manager.InterceptInfo.HeadersEntry - nil, // 59: telepresence.manager.InterceptInfo.MetadataEntry - nil, // 60: telepresence.manager.InterceptInfo.EnvironmentEntry - nil, // 61: telepresence.manager.ReviewInterceptRequest.HeadersEntry - nil, // 62: telepresence.manager.ReviewInterceptRequest.MetadataEntry - nil, // 63: telepresence.manager.ReviewInterceptRequest.EnvironmentEntry - nil, // 64: telepresence.manager.LogsResponse.PodLogsEntry - nil, // 65: telepresence.manager.LogsResponse.PodYamlEntry - (*WorkloadInfo_Intercept)(nil), // 66: telepresence.manager.WorkloadInfo.Intercept - (*timestamppb.Timestamp)(nil), // 67: google.protobuf.Timestamp - (*durationpb.Duration)(nil), // 68: google.protobuf.Duration - (*emptypb.Empty)(nil), // 69: google.protobuf.Empty + (*UninstallAgentsRequest)(nil), // 53: telepresence.manager.UninstallAgentsRequest + (*AgentInfo_Mechanism)(nil), // 54: telepresence.manager.AgentInfo.Mechanism + (*AgentInfo_ContainerInfo)(nil), // 55: telepresence.manager.AgentInfo.ContainerInfo + nil, // 56: telepresence.manager.AgentInfo.ContainersEntry + nil, // 57: telepresence.manager.AgentInfo.ContainerInfo.EnvironmentEntry + nil, // 58: telepresence.manager.PreviewSpec.AddRequestHeadersEntry + nil, // 59: telepresence.manager.InterceptInfo.HeadersEntry + nil, // 60: telepresence.manager.InterceptInfo.MetadataEntry + nil, // 61: telepresence.manager.InterceptInfo.EnvironmentEntry + nil, // 62: telepresence.manager.ReviewInterceptRequest.HeadersEntry + nil, // 63: telepresence.manager.ReviewInterceptRequest.MetadataEntry + nil, // 64: telepresence.manager.ReviewInterceptRequest.EnvironmentEntry + nil, // 65: telepresence.manager.LogsResponse.PodLogsEntry + nil, // 66: telepresence.manager.LogsResponse.PodYamlEntry + (*WorkloadInfo_Intercept)(nil), // 67: telepresence.manager.WorkloadInfo.Intercept + (*timestamppb.Timestamp)(nil), // 68: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 69: google.protobuf.Duration + (*emptypb.Empty)(nil), // 70: google.protobuf.Empty } var file_manager_manager_proto_depIdxs = []int32{ - 53, // 0: telepresence.manager.AgentInfo.mechanisms:type_name -> telepresence.manager.AgentInfo.Mechanism - 55, // 1: telepresence.manager.AgentInfo.containers:type_name -> telepresence.manager.AgentInfo.ContainersEntry + 54, // 0: telepresence.manager.AgentInfo.mechanisms:type_name -> telepresence.manager.AgentInfo.Mechanism + 56, // 1: telepresence.manager.AgentInfo.containers:type_name -> telepresence.manager.AgentInfo.ContainersEntry 9, // 2: telepresence.manager.PreviewSpec.ingress:type_name -> telepresence.manager.IngressInfo - 57, // 3: telepresence.manager.PreviewSpec.add_request_headers:type_name -> telepresence.manager.PreviewSpec.AddRequestHeadersEntry + 58, // 3: telepresence.manager.PreviewSpec.add_request_headers:type_name -> telepresence.manager.PreviewSpec.AddRequestHeadersEntry 8, // 4: telepresence.manager.InterceptInfo.spec:type_name -> telepresence.manager.InterceptSpec 12, // 5: telepresence.manager.InterceptInfo.client_session:type_name -> telepresence.manager.SessionInfo 10, // 6: telepresence.manager.InterceptInfo.preview_spec:type_name -> telepresence.manager.PreviewSpec 0, // 7: telepresence.manager.InterceptInfo.disposition:type_name -> telepresence.manager.InterceptDispositionType - 58, // 8: telepresence.manager.InterceptInfo.headers:type_name -> telepresence.manager.InterceptInfo.HeadersEntry - 59, // 9: telepresence.manager.InterceptInfo.metadata:type_name -> telepresence.manager.InterceptInfo.MetadataEntry - 60, // 10: telepresence.manager.InterceptInfo.environment:type_name -> telepresence.manager.InterceptInfo.EnvironmentEntry - 67, // 11: telepresence.manager.InterceptInfo.modified_at:type_name -> google.protobuf.Timestamp + 59, // 8: telepresence.manager.InterceptInfo.headers:type_name -> telepresence.manager.InterceptInfo.HeadersEntry + 60, // 9: telepresence.manager.InterceptInfo.metadata:type_name -> telepresence.manager.InterceptInfo.MetadataEntry + 61, // 10: telepresence.manager.InterceptInfo.environment:type_name -> telepresence.manager.InterceptInfo.EnvironmentEntry + 68, // 11: telepresence.manager.InterceptInfo.modified_at:type_name -> google.protobuf.Timestamp 12, // 12: telepresence.manager.AgentsRequest.session:type_name -> telepresence.manager.SessionInfo 6, // 13: telepresence.manager.AgentInfoSnapshot.agents:type_name -> telepresence.manager.AgentInfo 11, // 14: telepresence.manager.InterceptInfoSnapshot.intercepts:type_name -> telepresence.manager.InterceptInfo @@ -5016,13 +5085,13 @@ var file_manager_manager_proto_depIdxs = []int32{ 12, // 21: telepresence.manager.GetInterceptRequest.session:type_name -> telepresence.manager.SessionInfo 12, // 22: telepresence.manager.ReviewInterceptRequest.session:type_name -> telepresence.manager.SessionInfo 0, // 23: telepresence.manager.ReviewInterceptRequest.disposition:type_name -> telepresence.manager.InterceptDispositionType - 61, // 24: telepresence.manager.ReviewInterceptRequest.headers:type_name -> telepresence.manager.ReviewInterceptRequest.HeadersEntry - 62, // 25: telepresence.manager.ReviewInterceptRequest.metadata:type_name -> telepresence.manager.ReviewInterceptRequest.MetadataEntry - 63, // 26: telepresence.manager.ReviewInterceptRequest.environment:type_name -> telepresence.manager.ReviewInterceptRequest.EnvironmentEntry + 62, // 24: telepresence.manager.ReviewInterceptRequest.headers:type_name -> telepresence.manager.ReviewInterceptRequest.HeadersEntry + 63, // 25: telepresence.manager.ReviewInterceptRequest.metadata:type_name -> telepresence.manager.ReviewInterceptRequest.MetadataEntry + 64, // 26: telepresence.manager.ReviewInterceptRequest.environment:type_name -> telepresence.manager.ReviewInterceptRequest.EnvironmentEntry 12, // 27: telepresence.manager.RemainRequest.session:type_name -> telepresence.manager.SessionInfo - 68, // 28: telepresence.manager.LogLevelRequest.duration:type_name -> google.protobuf.Duration - 64, // 29: telepresence.manager.LogsResponse.pod_logs:type_name -> telepresence.manager.LogsResponse.PodLogsEntry - 65, // 30: telepresence.manager.LogsResponse.pod_yaml:type_name -> telepresence.manager.LogsResponse.PodYamlEntry + 69, // 28: telepresence.manager.LogLevelRequest.duration:type_name -> google.protobuf.Duration + 65, // 29: telepresence.manager.LogsResponse.pod_logs:type_name -> telepresence.manager.LogsResponse.PodLogsEntry + 66, // 30: telepresence.manager.LogsResponse.pod_yaml:type_name -> telepresence.manager.LogsResponse.PodYamlEntry 12, // 31: telepresence.manager.DNSRequest.session:type_name -> telepresence.manager.SessionInfo 12, // 32: telepresence.manager.DNSAgentResponse.session:type_name -> telepresence.manager.SessionInfo 34, // 33: telepresence.manager.DNSAgentResponse.request:type_name -> telepresence.manager.DNSRequest @@ -5039,91 +5108,94 @@ var file_manager_manager_proto_depIdxs = []int32{ 1, // 44: telepresence.manager.KnownWorkloadKinds.kinds:type_name -> telepresence.manager.WorkloadInfo.Kind 1, // 45: telepresence.manager.WorkloadInfo.kind:type_name -> telepresence.manager.WorkloadInfo.Kind 3, // 46: telepresence.manager.WorkloadInfo.agent_state:type_name -> telepresence.manager.WorkloadInfo.AgentState - 66, // 47: telepresence.manager.WorkloadInfo.intercept_clients:type_name -> telepresence.manager.WorkloadInfo.Intercept + 67, // 47: telepresence.manager.WorkloadInfo.intercept_clients:type_name -> telepresence.manager.WorkloadInfo.Intercept 2, // 48: telepresence.manager.WorkloadInfo.state:type_name -> telepresence.manager.WorkloadInfo.State 4, // 49: telepresence.manager.WorkloadEvent.type:type_name -> telepresence.manager.WorkloadEvent.Type 49, // 50: telepresence.manager.WorkloadEvent.workload:type_name -> telepresence.manager.WorkloadInfo - 67, // 51: telepresence.manager.WorkloadEventsDelta.since:type_name -> google.protobuf.Timestamp + 68, // 51: telepresence.manager.WorkloadEventsDelta.since:type_name -> google.protobuf.Timestamp 50, // 52: telepresence.manager.WorkloadEventsDelta.events:type_name -> telepresence.manager.WorkloadEvent 12, // 53: telepresence.manager.WorkloadEventsRequest.session_info:type_name -> telepresence.manager.SessionInfo - 67, // 54: telepresence.manager.WorkloadEventsRequest.since:type_name -> google.protobuf.Timestamp - 56, // 55: telepresence.manager.AgentInfo.ContainerInfo.environment:type_name -> telepresence.manager.AgentInfo.ContainerInfo.EnvironmentEntry - 54, // 56: telepresence.manager.AgentInfo.ContainersEntry.value:type_name -> telepresence.manager.AgentInfo.ContainerInfo - 69, // 57: telepresence.manager.Manager.Version:input_type -> google.protobuf.Empty - 69, // 58: telepresence.manager.Manager.GetAgentImageFQN:input_type -> google.protobuf.Empty - 45, // 59: telepresence.manager.Manager.GetAgentConfig:input_type -> telepresence.manager.AgentConfigRequest - 69, // 60: telepresence.manager.Manager.GetLicense:input_type -> google.protobuf.Empty - 69, // 61: telepresence.manager.Manager.CanConnectAmbassadorCloud:input_type -> google.protobuf.Empty - 69, // 62: telepresence.manager.Manager.GetCloudConfig:input_type -> google.protobuf.Empty - 69, // 63: telepresence.manager.Manager.GetClientConfig:input_type -> google.protobuf.Empty - 69, // 64: telepresence.manager.Manager.GetTelepresenceAPI:input_type -> google.protobuf.Empty - 5, // 65: telepresence.manager.Manager.ArriveAsClient:input_type -> telepresence.manager.ClientInfo - 6, // 66: telepresence.manager.Manager.ArriveAsAgent:input_type -> telepresence.manager.AgentInfo - 23, // 67: telepresence.manager.Manager.Remain:input_type -> telepresence.manager.RemainRequest - 12, // 68: telepresence.manager.Manager.Depart:input_type -> telepresence.manager.SessionInfo - 24, // 69: telepresence.manager.Manager.SetLogLevel:input_type -> telepresence.manager.LogLevelRequest - 25, // 70: telepresence.manager.Manager.GetLogs:input_type -> telepresence.manager.GetLogsRequest - 12, // 71: telepresence.manager.Manager.WatchAgentPods:input_type -> telepresence.manager.SessionInfo - 12, // 72: telepresence.manager.Manager.WatchAgents:input_type -> telepresence.manager.SessionInfo - 13, // 73: telepresence.manager.Manager.WatchAgentsNS:input_type -> telepresence.manager.AgentsRequest - 12, // 74: telepresence.manager.Manager.WatchIntercepts:input_type -> telepresence.manager.SessionInfo - 52, // 75: telepresence.manager.Manager.WatchWorkloads:input_type -> telepresence.manager.WorkloadEventsRequest - 12, // 76: telepresence.manager.Manager.WatchClusterInfo:input_type -> telepresence.manager.SessionInfo - 17, // 77: telepresence.manager.Manager.EnsureAgent:input_type -> telepresence.manager.EnsureAgentRequest - 16, // 78: telepresence.manager.Manager.PrepareIntercept:input_type -> telepresence.manager.CreateInterceptRequest - 16, // 79: telepresence.manager.Manager.CreateIntercept:input_type -> telepresence.manager.CreateInterceptRequest - 20, // 80: telepresence.manager.Manager.RemoveIntercept:input_type -> telepresence.manager.RemoveInterceptRequest2 - 19, // 81: telepresence.manager.Manager.UpdateIntercept:input_type -> telepresence.manager.UpdateInterceptRequest - 21, // 82: telepresence.manager.Manager.GetIntercept:input_type -> telepresence.manager.GetInterceptRequest - 22, // 83: telepresence.manager.Manager.ReviewIntercept:input_type -> telepresence.manager.ReviewInterceptRequest - 12, // 84: telepresence.manager.Manager.GetKnownWorkloadKinds:input_type -> telepresence.manager.SessionInfo - 34, // 85: telepresence.manager.Manager.LookupDNS:input_type -> telepresence.manager.DNSRequest - 36, // 86: telepresence.manager.Manager.AgentLookupDNSResponse:input_type -> telepresence.manager.DNSAgentResponse - 12, // 87: telepresence.manager.Manager.WatchLookupDNS:input_type -> telepresence.manager.SessionInfo - 69, // 88: telepresence.manager.Manager.WatchLogLevel:input_type -> google.protobuf.Empty - 32, // 89: telepresence.manager.Manager.Tunnel:input_type -> telepresence.manager.TunnelMessage - 47, // 90: telepresence.manager.Manager.ReportMetrics:input_type -> telepresence.manager.TunnelMetrics - 12, // 91: telepresence.manager.Manager.WatchDial:input_type -> telepresence.manager.SessionInfo - 28, // 92: telepresence.manager.Manager.Version:output_type -> telepresence.manager.VersionInfo2 - 42, // 93: telepresence.manager.Manager.GetAgentImageFQN:output_type -> telepresence.manager.AgentImageFQN - 46, // 94: telepresence.manager.Manager.GetAgentConfig:output_type -> telepresence.manager.AgentConfigResponse - 29, // 95: telepresence.manager.Manager.GetLicense:output_type -> telepresence.manager.License - 31, // 96: telepresence.manager.Manager.CanConnectAmbassadorCloud:output_type -> telepresence.manager.AmbassadorCloudConnection - 30, // 97: telepresence.manager.Manager.GetCloudConfig:output_type -> telepresence.manager.AmbassadorCloudConfig - 41, // 98: telepresence.manager.Manager.GetClientConfig:output_type -> telepresence.manager.CLIConfig - 27, // 99: telepresence.manager.Manager.GetTelepresenceAPI:output_type -> telepresence.manager.TelepresenceAPIInfo - 12, // 100: telepresence.manager.Manager.ArriveAsClient:output_type -> telepresence.manager.SessionInfo - 12, // 101: telepresence.manager.Manager.ArriveAsAgent:output_type -> telepresence.manager.SessionInfo - 69, // 102: telepresence.manager.Manager.Remain:output_type -> google.protobuf.Empty - 69, // 103: telepresence.manager.Manager.Depart:output_type -> google.protobuf.Empty - 69, // 104: telepresence.manager.Manager.SetLogLevel:output_type -> google.protobuf.Empty - 26, // 105: telepresence.manager.Manager.GetLogs:output_type -> telepresence.manager.LogsResponse - 44, // 106: telepresence.manager.Manager.WatchAgentPods:output_type -> telepresence.manager.AgentPodInfoSnapshot - 14, // 107: telepresence.manager.Manager.WatchAgents:output_type -> telepresence.manager.AgentInfoSnapshot - 14, // 108: telepresence.manager.Manager.WatchAgentsNS:output_type -> telepresence.manager.AgentInfoSnapshot - 15, // 109: telepresence.manager.Manager.WatchIntercepts:output_type -> telepresence.manager.InterceptInfoSnapshot - 51, // 110: telepresence.manager.Manager.WatchWorkloads:output_type -> telepresence.manager.WorkloadEventsDelta - 38, // 111: telepresence.manager.Manager.WatchClusterInfo:output_type -> telepresence.manager.ClusterInfo - 14, // 112: telepresence.manager.Manager.EnsureAgent:output_type -> telepresence.manager.AgentInfoSnapshot - 18, // 113: telepresence.manager.Manager.PrepareIntercept:output_type -> telepresence.manager.PreparedIntercept - 11, // 114: telepresence.manager.Manager.CreateIntercept:output_type -> telepresence.manager.InterceptInfo - 69, // 115: telepresence.manager.Manager.RemoveIntercept:output_type -> google.protobuf.Empty - 11, // 116: telepresence.manager.Manager.UpdateIntercept:output_type -> telepresence.manager.InterceptInfo - 11, // 117: telepresence.manager.Manager.GetIntercept:output_type -> telepresence.manager.InterceptInfo - 69, // 118: telepresence.manager.Manager.ReviewIntercept:output_type -> google.protobuf.Empty - 48, // 119: telepresence.manager.Manager.GetKnownWorkloadKinds:output_type -> telepresence.manager.KnownWorkloadKinds - 35, // 120: telepresence.manager.Manager.LookupDNS:output_type -> telepresence.manager.DNSResponse - 69, // 121: telepresence.manager.Manager.AgentLookupDNSResponse:output_type -> google.protobuf.Empty - 34, // 122: telepresence.manager.Manager.WatchLookupDNS:output_type -> telepresence.manager.DNSRequest - 24, // 123: telepresence.manager.Manager.WatchLogLevel:output_type -> telepresence.manager.LogLevelRequest - 32, // 124: telepresence.manager.Manager.Tunnel:output_type -> telepresence.manager.TunnelMessage - 69, // 125: telepresence.manager.Manager.ReportMetrics:output_type -> google.protobuf.Empty - 33, // 126: telepresence.manager.Manager.WatchDial:output_type -> telepresence.manager.DialRequest - 92, // [92:127] is the sub-list for method output_type - 57, // [57:92] is the sub-list for method input_type - 57, // [57:57] is the sub-list for extension type_name - 57, // [57:57] is the sub-list for extension extendee - 0, // [0:57] is the sub-list for field type_name + 68, // 54: telepresence.manager.WorkloadEventsRequest.since:type_name -> google.protobuf.Timestamp + 12, // 55: telepresence.manager.UninstallAgentsRequest.session_info:type_name -> telepresence.manager.SessionInfo + 57, // 56: telepresence.manager.AgentInfo.ContainerInfo.environment:type_name -> telepresence.manager.AgentInfo.ContainerInfo.EnvironmentEntry + 55, // 57: telepresence.manager.AgentInfo.ContainersEntry.value:type_name -> telepresence.manager.AgentInfo.ContainerInfo + 70, // 58: telepresence.manager.Manager.Version:input_type -> google.protobuf.Empty + 70, // 59: telepresence.manager.Manager.GetAgentImageFQN:input_type -> google.protobuf.Empty + 45, // 60: telepresence.manager.Manager.GetAgentConfig:input_type -> telepresence.manager.AgentConfigRequest + 70, // 61: telepresence.manager.Manager.GetLicense:input_type -> google.protobuf.Empty + 70, // 62: telepresence.manager.Manager.CanConnectAmbassadorCloud:input_type -> google.protobuf.Empty + 70, // 63: telepresence.manager.Manager.GetCloudConfig:input_type -> google.protobuf.Empty + 70, // 64: telepresence.manager.Manager.GetClientConfig:input_type -> google.protobuf.Empty + 70, // 65: telepresence.manager.Manager.GetTelepresenceAPI:input_type -> google.protobuf.Empty + 5, // 66: telepresence.manager.Manager.ArriveAsClient:input_type -> telepresence.manager.ClientInfo + 6, // 67: telepresence.manager.Manager.ArriveAsAgent:input_type -> telepresence.manager.AgentInfo + 23, // 68: telepresence.manager.Manager.Remain:input_type -> telepresence.manager.RemainRequest + 12, // 69: telepresence.manager.Manager.Depart:input_type -> telepresence.manager.SessionInfo + 24, // 70: telepresence.manager.Manager.SetLogLevel:input_type -> telepresence.manager.LogLevelRequest + 25, // 71: telepresence.manager.Manager.GetLogs:input_type -> telepresence.manager.GetLogsRequest + 12, // 72: telepresence.manager.Manager.WatchAgentPods:input_type -> telepresence.manager.SessionInfo + 12, // 73: telepresence.manager.Manager.WatchAgents:input_type -> telepresence.manager.SessionInfo + 13, // 74: telepresence.manager.Manager.WatchAgentsNS:input_type -> telepresence.manager.AgentsRequest + 12, // 75: telepresence.manager.Manager.WatchIntercepts:input_type -> telepresence.manager.SessionInfo + 52, // 76: telepresence.manager.Manager.WatchWorkloads:input_type -> telepresence.manager.WorkloadEventsRequest + 12, // 77: telepresence.manager.Manager.WatchClusterInfo:input_type -> telepresence.manager.SessionInfo + 17, // 78: telepresence.manager.Manager.EnsureAgent:input_type -> telepresence.manager.EnsureAgentRequest + 16, // 79: telepresence.manager.Manager.PrepareIntercept:input_type -> telepresence.manager.CreateInterceptRequest + 16, // 80: telepresence.manager.Manager.CreateIntercept:input_type -> telepresence.manager.CreateInterceptRequest + 20, // 81: telepresence.manager.Manager.RemoveIntercept:input_type -> telepresence.manager.RemoveInterceptRequest2 + 19, // 82: telepresence.manager.Manager.UpdateIntercept:input_type -> telepresence.manager.UpdateInterceptRequest + 21, // 83: telepresence.manager.Manager.GetIntercept:input_type -> telepresence.manager.GetInterceptRequest + 22, // 84: telepresence.manager.Manager.ReviewIntercept:input_type -> telepresence.manager.ReviewInterceptRequest + 12, // 85: telepresence.manager.Manager.GetKnownWorkloadKinds:input_type -> telepresence.manager.SessionInfo + 34, // 86: telepresence.manager.Manager.LookupDNS:input_type -> telepresence.manager.DNSRequest + 36, // 87: telepresence.manager.Manager.AgentLookupDNSResponse:input_type -> telepresence.manager.DNSAgentResponse + 12, // 88: telepresence.manager.Manager.WatchLookupDNS:input_type -> telepresence.manager.SessionInfo + 70, // 89: telepresence.manager.Manager.WatchLogLevel:input_type -> google.protobuf.Empty + 32, // 90: telepresence.manager.Manager.Tunnel:input_type -> telepresence.manager.TunnelMessage + 47, // 91: telepresence.manager.Manager.ReportMetrics:input_type -> telepresence.manager.TunnelMetrics + 12, // 92: telepresence.manager.Manager.WatchDial:input_type -> telepresence.manager.SessionInfo + 53, // 93: telepresence.manager.Manager.UninstallAgents:input_type -> telepresence.manager.UninstallAgentsRequest + 28, // 94: telepresence.manager.Manager.Version:output_type -> telepresence.manager.VersionInfo2 + 42, // 95: telepresence.manager.Manager.GetAgentImageFQN:output_type -> telepresence.manager.AgentImageFQN + 46, // 96: telepresence.manager.Manager.GetAgentConfig:output_type -> telepresence.manager.AgentConfigResponse + 29, // 97: telepresence.manager.Manager.GetLicense:output_type -> telepresence.manager.License + 31, // 98: telepresence.manager.Manager.CanConnectAmbassadorCloud:output_type -> telepresence.manager.AmbassadorCloudConnection + 30, // 99: telepresence.manager.Manager.GetCloudConfig:output_type -> telepresence.manager.AmbassadorCloudConfig + 41, // 100: telepresence.manager.Manager.GetClientConfig:output_type -> telepresence.manager.CLIConfig + 27, // 101: telepresence.manager.Manager.GetTelepresenceAPI:output_type -> telepresence.manager.TelepresenceAPIInfo + 12, // 102: telepresence.manager.Manager.ArriveAsClient:output_type -> telepresence.manager.SessionInfo + 12, // 103: telepresence.manager.Manager.ArriveAsAgent:output_type -> telepresence.manager.SessionInfo + 70, // 104: telepresence.manager.Manager.Remain:output_type -> google.protobuf.Empty + 70, // 105: telepresence.manager.Manager.Depart:output_type -> google.protobuf.Empty + 70, // 106: telepresence.manager.Manager.SetLogLevel:output_type -> google.protobuf.Empty + 26, // 107: telepresence.manager.Manager.GetLogs:output_type -> telepresence.manager.LogsResponse + 44, // 108: telepresence.manager.Manager.WatchAgentPods:output_type -> telepresence.manager.AgentPodInfoSnapshot + 14, // 109: telepresence.manager.Manager.WatchAgents:output_type -> telepresence.manager.AgentInfoSnapshot + 14, // 110: telepresence.manager.Manager.WatchAgentsNS:output_type -> telepresence.manager.AgentInfoSnapshot + 15, // 111: telepresence.manager.Manager.WatchIntercepts:output_type -> telepresence.manager.InterceptInfoSnapshot + 51, // 112: telepresence.manager.Manager.WatchWorkloads:output_type -> telepresence.manager.WorkloadEventsDelta + 38, // 113: telepresence.manager.Manager.WatchClusterInfo:output_type -> telepresence.manager.ClusterInfo + 14, // 114: telepresence.manager.Manager.EnsureAgent:output_type -> telepresence.manager.AgentInfoSnapshot + 18, // 115: telepresence.manager.Manager.PrepareIntercept:output_type -> telepresence.manager.PreparedIntercept + 11, // 116: telepresence.manager.Manager.CreateIntercept:output_type -> telepresence.manager.InterceptInfo + 70, // 117: telepresence.manager.Manager.RemoveIntercept:output_type -> google.protobuf.Empty + 11, // 118: telepresence.manager.Manager.UpdateIntercept:output_type -> telepresence.manager.InterceptInfo + 11, // 119: telepresence.manager.Manager.GetIntercept:output_type -> telepresence.manager.InterceptInfo + 70, // 120: telepresence.manager.Manager.ReviewIntercept:output_type -> google.protobuf.Empty + 48, // 121: telepresence.manager.Manager.GetKnownWorkloadKinds:output_type -> telepresence.manager.KnownWorkloadKinds + 35, // 122: telepresence.manager.Manager.LookupDNS:output_type -> telepresence.manager.DNSResponse + 70, // 123: telepresence.manager.Manager.AgentLookupDNSResponse:output_type -> google.protobuf.Empty + 34, // 124: telepresence.manager.Manager.WatchLookupDNS:output_type -> telepresence.manager.DNSRequest + 24, // 125: telepresence.manager.Manager.WatchLogLevel:output_type -> telepresence.manager.LogLevelRequest + 32, // 126: telepresence.manager.Manager.Tunnel:output_type -> telepresence.manager.TunnelMessage + 70, // 127: telepresence.manager.Manager.ReportMetrics:output_type -> google.protobuf.Empty + 33, // 128: telepresence.manager.Manager.WatchDial:output_type -> telepresence.manager.DialRequest + 70, // 129: telepresence.manager.Manager.UninstallAgents:output_type -> google.protobuf.Empty + 94, // [94:130] is the sub-list for method output_type + 58, // [58:94] is the sub-list for method input_type + 58, // [58:58] is the sub-list for extension type_name + 58, // [58:58] is the sub-list for extension extendee + 0, // [0:58] is the sub-list for field type_name } func init() { file_manager_manager_proto_init() } @@ -5143,7 +5215,7 @@ func file_manager_manager_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_manager_manager_proto_rawDesc), len(file_manager_manager_proto_rawDesc)), NumEnums: 5, - NumMessages: 62, + NumMessages: 63, NumExtensions: 0, NumServices: 1, }, diff --git a/rpc/manager/manager.proto b/rpc/manager/manager.proto index 28a198a515..f852193f94 100644 --- a/rpc/manager/manager.proto +++ b/rpc/manager/manager.proto @@ -714,6 +714,15 @@ message WorkloadEventsRequest { string namespace = 3; } +message UninstallAgentsRequest { + // The session_info identifies the client connection, and hence the + // namespace for the resulting watcher. + SessionInfo session_info = 1; + + // The agents to install. Empty means all agents in the connected namespace. + repeated string agents = 2; +} + service Manager { // Version returns the version information of the Manager. rpc Version(google.protobuf.Empty) returns (VersionInfo2); @@ -861,4 +870,8 @@ service Manager { // connection and responds with a Tunnel. The manager then connects the // two tunnels. rpc WatchDial(SessionInfo) returns (stream DialRequest); + + // UninstallAgents will uninstall the traffic-agent from the given workloads (or all + // workloads if the list is empty). + rpc UninstallAgents(UninstallAgentsRequest) returns (google.protobuf.Empty); } diff --git a/rpc/manager/manager_grpc.pb.go b/rpc/manager/manager_grpc.pb.go index 784cdeef69..c9f756bc6b 100644 --- a/rpc/manager/manager_grpc.pb.go +++ b/rpc/manager/manager_grpc.pb.go @@ -59,6 +59,7 @@ const ( Manager_Tunnel_FullMethodName = "/telepresence.manager.Manager/Tunnel" Manager_ReportMetrics_FullMethodName = "/telepresence.manager.Manager/ReportMetrics" Manager_WatchDial_FullMethodName = "/telepresence.manager.Manager/WatchDial" + Manager_UninstallAgents_FullMethodName = "/telepresence.manager.Manager/UninstallAgents" ) // ManagerClient is the client API for Manager service. @@ -177,6 +178,9 @@ type ManagerClient interface { // connection and responds with a Tunnel. The manager then connects the // two tunnels. WatchDial(ctx context.Context, in *SessionInfo, opts ...grpc.CallOption) (grpc.ServerStreamingClient[DialRequest], error) + // UninstallAgents will uninstall the traffic-agent from the given workloads (or all + // workloads if the list is empty). + UninstallAgents(ctx context.Context, in *UninstallAgentsRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) } type managerClient struct { @@ -621,6 +625,16 @@ func (c *managerClient) WatchDial(ctx context.Context, in *SessionInfo, opts ... // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type Manager_WatchDialClient = grpc.ServerStreamingClient[DialRequest] +func (c *managerClient) UninstallAgents(ctx context.Context, in *UninstallAgentsRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, Manager_UninstallAgents_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // ManagerServer is the server API for Manager service. // All implementations must embed UnimplementedManagerServer // for forward compatibility. @@ -737,6 +751,9 @@ type ManagerServer interface { // connection and responds with a Tunnel. The manager then connects the // two tunnels. WatchDial(*SessionInfo, grpc.ServerStreamingServer[DialRequest]) error + // UninstallAgents will uninstall the traffic-agent from the given workloads (or all + // workloads if the list is empty). + UninstallAgents(context.Context, *UninstallAgentsRequest) (*emptypb.Empty, error) mustEmbedUnimplementedManagerServer() } @@ -852,6 +869,9 @@ func (UnimplementedManagerServer) ReportMetrics(context.Context, *TunnelMetrics) func (UnimplementedManagerServer) WatchDial(*SessionInfo, grpc.ServerStreamingServer[DialRequest]) error { return status.Errorf(codes.Unimplemented, "method WatchDial not implemented") } +func (UnimplementedManagerServer) UninstallAgents(context.Context, *UninstallAgentsRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method UninstallAgents not implemented") +} func (UnimplementedManagerServer) mustEmbedUnimplementedManagerServer() {} func (UnimplementedManagerServer) testEmbeddedByValue() {} @@ -1429,6 +1449,24 @@ func _Manager_WatchDial_Handler(srv interface{}, stream grpc.ServerStream) error // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type Manager_WatchDialServer = grpc.ServerStreamingServer[DialRequest] +func _Manager_UninstallAgents_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UninstallAgentsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ManagerServer).UninstallAgents(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Manager_UninstallAgents_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ManagerServer).UninstallAgents(ctx, req.(*UninstallAgentsRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Manager_ServiceDesc is the grpc.ServiceDesc for Manager service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -1536,6 +1574,10 @@ var Manager_ServiceDesc = grpc.ServiceDesc{ MethodName: "ReportMetrics", Handler: _Manager_ReportMetrics_Handler, }, + { + MethodName: "UninstallAgents", + Handler: _Manager_UninstallAgents_Handler, + }, }, Streams: []grpc.StreamDesc{ { From cf71a41c8bcf607c23a7918ed8fcd08040794308 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 28 Jan 2025 07:54:20 +0100 Subject: [PATCH 21/61] Use Serve function of grpc.Server in favor of dhttp.ServerConfig. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/agent/agent.go | 38 ++++++---- cmd/traffic/cmd/manager/manager.go | 35 +++------ pkg/authenticator/authenticator.go | 15 ++-- pkg/authenticator/grpc/authenticator.go | 6 +- pkg/client/docker/kubeauth/cmd.go | 45 +++++++----- pkg/client/k8s_config.go | 4 + pkg/client/rootd/service.go | 17 +---- pkg/client/userd/daemon/service.go | 41 +++++------ pkg/client/userd/session.go | 2 + pkg/client/userd/trafficmgr/session.go | 8 +- pkg/grpc/server/server.go | 97 +++++++++++++++++++++++++ pkg/k8sapi/clientconfigprovider.go | 7 ++ 12 files changed, 207 insertions(+), 108 deletions(-) create mode 100644 pkg/grpc/server/server.go create mode 100644 pkg/k8sapi/clientconfigprovider.go diff --git a/cmd/traffic/cmd/agent/agent.go b/cmd/traffic/cmd/agent/agent.go index 500a70d637..670a430557 100644 --- a/cmd/traffic/cmd/agent/agent.go +++ b/cmd/traffic/cmd/agent/agent.go @@ -7,17 +7,19 @@ import ( "fmt" "io" "net" + "os" + "os/signal" "path/filepath" "strings" "time" "github.com/pkg/sftp" + "golang.org/x/sys/unix" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/datawire/dlib/dgroup" - "github.com/datawire/dlib/dhttp" "github.com/datawire/dlib/dlog" ftp "github.com/datawire/go-ftpserver" "github.com/telepresenceio/telepresence/rpc/v2/agent" @@ -25,6 +27,7 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/dos" "github.com/telepresenceio/telepresence/v2/pkg/forwarder" + "github.com/telepresenceio/telepresence/v2/pkg/grpc/server" "github.com/telepresenceio/telepresence/v2/pkg/iputil" "github.com/telepresenceio/telepresence/v2/pkg/restapi" "github.com/telepresenceio/telepresence/v2/pkg/tunnel" @@ -131,6 +134,22 @@ func sftpServer(ctx context.Context, sftpPortCh chan<- uint16) error { func Main(ctx context.Context, _ ...string) error { dlog.Infof(ctx, "Traffic Agent %s", version.Version) + ctx, cancel := context.WithCancel(ctx) + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, os.Interrupt, unix.SIGTERM) + defer func() { + signal.Stop(sigs) + cancel() + }() + + go func() { + select { + case <-sigs: + cancel() + case <-ctx.Done(): + } + }() + // Handle configuration config, err := LoadConfig(ctx) if err != nil { @@ -138,9 +157,8 @@ func Main(ctx context.Context, _ ...string) error { } g := dgroup.NewGroup(ctx, dgroup.GroupConfig{ - EnableSignalHandling: true, + SoftShutdownTimeout: 10 * time.Second, // Agent must be able to depart. }) - s := NewState(config) info, err := StartServices(ctx, g, config, s) if err != nil { @@ -246,22 +264,14 @@ func StartServices(ctx context.Context, g *dgroup.Group, config Config, srv Stat if err != nil { return err } - defer func() { - _ = grpcListener.Close() - }() grpcAddress := grpcListener.Addr().(*net.TCPAddr) grpcPortCh <- uint16(grpcAddress.Port) dlog.Debugf(ctx, "Listener opened on %s", grpcAddress) - grpcHandler := grpc.NewServer(grpcOpts...) - agent.RegisterAgentServer(grpcHandler, srv) - sc := &dhttp.ServerConfig{Handler: grpcHandler} - dlog.Info(ctx, "gRPC server started") - if err = sc.Serve(ctx, grpcListener); err != nil && ctx.Err() != nil { - err = nil // Normal shutdown - } - return err + svc := server.New(ctx, grpcOpts...) + agent.RegisterAgentServer(svc, srv) + return server.Serve(ctx, svc, grpcListener) }) sftpPortCh := make(chan uint16) diff --git a/cmd/traffic/cmd/manager/manager.go b/cmd/traffic/cmd/manager/manager.go index 7d2ba50b0d..a69eb96f97 100644 --- a/cmd/traffic/cmd/manager/manager.go +++ b/cmd/traffic/cmd/manager/manager.go @@ -3,10 +3,10 @@ package manager import ( "context" "fmt" - "net/http" + "net" "os" "slices" - "strings" + "strconv" "sync/atomic" "time" @@ -28,8 +28,8 @@ import ( "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/mutator" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/namespaces" "github.com/telepresenceio/telepresence/v2/pkg/agentmap" + "github.com/telepresenceio/telepresence/v2/pkg/grpc/server" "github.com/telepresenceio/telepresence/v2/pkg/informer" - "github.com/telepresenceio/telepresence/v2/pkg/ioutil" "github.com/telepresenceio/telepresence/v2/pkg/iputil" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" "github.com/telepresenceio/telepresence/v2/pkg/version" @@ -270,16 +270,16 @@ func (s *service) serveHTTP(ctx context.Context) error { env := managerutil.GetEnv(ctx) host := env.ServerHost port := env.ServerPort + l, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(int(port)))) + if err != nil { + return err + } + var opts []grpc.ServerOption if mz, ok := env.MaxReceiveSize.AsInt64(); ok { opts = append(opts, grpc.MaxRecvMsgSize(int(mz))) } - grpcHandler := grpc.NewServer(opts...) - httpHandler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ioutil.Printf(w, "Hello World from: %s\n", r.URL.Path) - })) - lg := dlog.StdLogger(ctx, dlog.MaxLogLevel(ctx)) addr := iputil.JoinHostPort(host, port) if host == "" { @@ -287,22 +287,9 @@ func (s *service) serveHTTP(ctx context.Context) error { } else { lg.SetPrefix(fmt.Sprintf("grpc-api %s", addr)) } - sc := &dhttp.ServerConfig{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.ProtoMajor == 2 && strings.HasPrefix(r.Header.Get("Content-Type"), "application/grpc") { - atomic.AddInt32(&s.activeGrpcRequests, 1) - grpcHandler.ServeHTTP(w, r) - atomic.AddInt32(&s.activeGrpcRequests, -1) - } else { - atomic.AddInt32(&s.activeHttpRequests, 1) - httpHandler.ServeHTTP(w, r) - atomic.AddInt32(&s.activeHttpRequests, -1) - } - }), - ErrorLog: lg, - } - s.self.RegisterServers(grpcHandler) - return sc.ListenAndServe(ctx, fmt.Sprintf("%s:%d", host, port)) + svc := server.New(ctx, opts...) + s.self.RegisterServers(svc) + return server.Serve(ctx, svc, l) } func (s *service) RegisterServers(grpcHandler *grpc.Server) { diff --git a/pkg/authenticator/authenticator.go b/pkg/authenticator/authenticator.go index 84dc37abc9..9dc46b0e8f 100644 --- a/pkg/authenticator/authenticator.go +++ b/pkg/authenticator/authenticator.go @@ -4,15 +4,16 @@ import ( "context" "fmt" - "k8s.io/client-go/tools/clientcmd" clientcmd_api "k8s.io/client-go/tools/clientcmd/api" + + "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" ) func NewService( - kubeClientConfig clientcmd.ClientConfig, + clientConfigProvider k8sapi.ClientConfigProvider, ) *Service { return &Service{ - kubeClientConfig: kubeClientConfig, + clientConfigProvider: clientConfigProvider, execCredentialsResolver: execCredentialBinary{}, } } @@ -27,7 +28,7 @@ type ExecCredentialsResolver interface { //go:generate go run go.uber.org/mock/mockgen -package=mock_authenticator -destination=mocks/clientconfig_mock.go k8s.io/client-go/tools/clientcmd ClientConfig type Service struct { - kubeClientConfig clientcmd.ClientConfig + clientConfigProvider k8sapi.ClientConfigProvider execCredentialsResolver ExecCredentialsResolver } @@ -46,7 +47,11 @@ func (a Service) GetExecCredentials(ctx context.Context, contextName string) ([] } func (a Service) getExecConfigFromContext(contextName string) (*clientcmd_api.ExecConfig, error) { - rawConfig, err := a.kubeClientConfig.RawConfig() + cc, err := a.clientConfigProvider.ClientConfig() + if err != nil { + return nil, fmt.Errorf("failed to get kubeconfig provider: %w", err) + } + rawConfig, err := cc.RawConfig() if err != nil { return nil, fmt.Errorf("failed to get kubeconfig: %w", err) } diff --git a/pkg/authenticator/grpc/authenticator.go b/pkg/authenticator/grpc/authenticator.go index b4a7bc8c7d..b16cbc33c5 100644 --- a/pkg/authenticator/grpc/authenticator.go +++ b/pkg/authenticator/grpc/authenticator.go @@ -5,16 +5,16 @@ import ( "fmt" "google.golang.org/grpc" - "k8s.io/client-go/tools/clientcmd" "github.com/datawire/dlib/dlog" rpc "github.com/telepresenceio/telepresence/rpc/v2/authenticator" "github.com/telepresenceio/telepresence/v2/pkg/authenticator" + "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" ) -func RegisterAuthenticatorServer(srv *grpc.Server, kubeClientConfig clientcmd.ClientConfig) { +func RegisterAuthenticatorServer(srv *grpc.Server, clientConfigProvider k8sapi.ClientConfigProvider) { rpc.RegisterAuthenticatorServer(srv, &AuthenticatorServer{ - authenticator: authenticator.NewService(kubeClientConfig), + authenticator: authenticator.NewService(clientConfigProvider), }) } diff --git a/pkg/client/docker/kubeauth/cmd.go b/pkg/client/docker/kubeauth/cmd.go index 52f952dfe3..cdc023f16a 100644 --- a/pkg/client/docker/kubeauth/cmd.go +++ b/pkg/client/docker/kubeauth/cmd.go @@ -12,16 +12,16 @@ import ( "github.com/fsnotify/fsnotify" "github.com/go-json-experiment/json" "github.com/spf13/cobra" - "google.golang.org/grpc" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/tools/clientcmd" "github.com/datawire/dlib/dgroup" - "github.com/datawire/dlib/dhttp" "github.com/datawire/dlib/dlog" authGrpc "github.com/telepresenceio/telepresence/v2/pkg/authenticator/grpc" "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/client/logging" "github.com/telepresenceio/telepresence/v2/pkg/errcat" + "github.com/telepresenceio/telepresence/v2/pkg/grpc/server" ) const ( @@ -31,10 +31,10 @@ const ( ) type authService struct { - portFile string - kubeFlags *genericclioptions.ConfigFlags - cancel context.CancelFunc - configFiles []string + portFile string + kubeFlags *genericclioptions.ConfigFlags + configFiles []string + clientConfig clientcmd.ClientConfig } type PortFile struct { @@ -80,8 +80,8 @@ func (as *authService) run(cmd *cobra.Command, _ []string) error { addr := grpcListener.Addr().(*net.TCPAddr) dlog.Infof(ctx, "kubeauth listening on address %s", addr) - config := as.kubeFlags.ToRawKubeConfigLoader() - as.configFiles = config.ConfigAccess().GetLoadingPrecedence() + as.clientConfig = as.kubeFlags.ToRawKubeConfigLoader() + as.configFiles = as.clientConfig.ConfigAccess().GetLoadingPrecedence() p := PortFile{ Port: addr.Port, Kubeconfig: strings.Join(as.configFiles, string(filepath.ListSeparator)), @@ -94,17 +94,28 @@ func (as *authService) run(cmd *cobra.Command, _ []string) error { return err } - ctx, as.cancel = context.WithCancel(ctx) - g := dgroup.NewGroup(ctx, dgroup.GroupConfig{}) + g := dgroup.NewGroup(ctx, dgroup.GroupConfig{ + EnableSignalHandling: true, + ShutdownOnNonError: true, + SoftShutdownTimeout: time.Second, + }) g.Go("portfile-alive", as.keepPortFileAlive) g.Go("portfile-watcher", as.watchFiles) g.Go("grpc-server", func(ctx context.Context) error { - grpcHandler := grpc.NewServer() - authGrpc.RegisterAuthenticatorServer(grpcHandler, config) - sc := &dhttp.ServerConfig{Handler: grpcHandler} - return sc.Serve(ctx, grpcListener) + svc := server.New(ctx) + authGrpc.RegisterAuthenticatorServer(svc, as) + return server.Serve(ctx, svc, grpcListener) }) - return g.Wait() + if err = g.Wait(); err != nil { + dlog.Errorf(ctx, "kubeauth exiting with error: %v", err) + } else { + dlog.Info(ctx, "kubeauth exiting") + } + return err +} + +func (as *authService) ClientConfig() (clientcmd.ClientConfig, error) { + return as.clientConfig, nil } func (as *authService) keepPortFileAlive(ctx context.Context) error { @@ -121,8 +132,6 @@ func (as *authService) keepPortFileAlive(ctx context.Context) error { return fmt.Errorf("failed to update timestamp on %s: %v", as.portFile, err) } // File is removed, so stop trying to update its timestamps and die - dlog.Info(ctx, "kubeauth exiting") - as.cancel() return nil } select { @@ -178,7 +187,7 @@ func (as *authService) watchFiles(ctx context.Context) error { case event := <-watcher.Events: if event.Op&(fsnotify.Remove|fsnotify.Write|fsnotify.Create) != 0 && isOfInterest(event.Name, files) { dlog.Infof(ctx, "Terminated due to %s in %s", event.Op, event.Name) - as.cancel() + return nil } } } diff --git a/pkg/client/k8s_config.go b/pkg/client/k8s_config.go index 357d799524..4cab0a5b32 100644 --- a/pkg/client/k8s_config.go +++ b/pkg/client/k8s_config.go @@ -664,6 +664,10 @@ func (kf *Kubeconfig) GetContext() string { return kf.Context } +func (kf *Kubeconfig) GetClientConfig() clientcmd.ClientConfig { + return kf.ClientConfig +} + func (kf *Kubeconfig) GetRestConfig() *rest.Config { return kf.RestConfig } diff --git a/pkg/client/rootd/service.go b/pkg/client/rootd/service.go index 9ea68746da..cc5a688a97 100644 --- a/pkg/client/rootd/service.go +++ b/pkg/client/rootd/service.go @@ -18,7 +18,6 @@ import ( "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dgroup" - "github.com/datawire/dlib/dhttp" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/rpc/v2/common" rpc "github.com/telepresenceio/telepresence/rpc/v2/daemon" @@ -29,6 +28,7 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/client/socket" "github.com/telepresenceio/telepresence/v2/pkg/errcat" "github.com/telepresenceio/telepresence/v2/pkg/filelocation" + "github.com/telepresenceio/telepresence/v2/pkg/grpc/server" "github.com/telepresenceio/telepresence/v2/pkg/log" "github.com/telepresenceio/telepresence/v2/pkg/pprof" "github.com/telepresenceio/telepresence/v2/pkg/proc" @@ -413,20 +413,9 @@ func (s *Service) serveGrpc(c context.Context, l net.Listener) error { if mz := cfg.Grpc().MaxReceiveSize(); mz > 0 { opts = append(opts, grpc.MaxRecvMsgSize(int(mz))) } - svc := grpc.NewServer(opts...) + svc := server.New(c, opts...) rpc.RegisterDaemonServer(svc, s) - - sc := &dhttp.ServerConfig{ - Handler: svc, - } - dlog.Info(c, "gRPC server started") - err := sc.Serve(c, l) - if err != nil { - dlog.Errorf(c, "gRPC server ended with: %v", err) - } else { - dlog.Debug(c, "gRPC server ended") - } - return err + return server.Serve(c, svc, l) } // run is the main function when executing as the daemon. diff --git a/pkg/client/userd/daemon/service.go b/pkg/client/userd/daemon/service.go index aaf2d78865..895b328524 100644 --- a/pkg/client/userd/daemon/service.go +++ b/pkg/client/userd/daemon/service.go @@ -16,12 +16,13 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "k8s.io/client-go/tools/clientcmd" "github.com/datawire/dlib/dgroup" - "github.com/datawire/dlib/dhttp" "github.com/datawire/dlib/dlog" rpc "github.com/telepresenceio/telepresence/rpc/v2/connector" "github.com/telepresenceio/telepresence/rpc/v2/manager" + authGrpc "github.com/telepresenceio/telepresence/v2/pkg/authenticator/grpc" "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/client/cli/daemon" "github.com/telepresenceio/telepresence/v2/pkg/client/logging" @@ -32,6 +33,7 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/client/userd/trafficmgr" "github.com/telepresenceio/telepresence/v2/pkg/errcat" "github.com/telepresenceio/telepresence/v2/pkg/filelocation" + "github.com/telepresenceio/telepresence/v2/pkg/grpc/server" "github.com/telepresenceio/telepresence/v2/pkg/log" "github.com/telepresenceio/telepresence/v2/pkg/pprof" "github.com/telepresenceio/telepresence/v2/pkg/proc" @@ -68,6 +70,7 @@ type service struct { // is in effect (rootSessionInProc == true). quitDisable bool + clientConfig clientcmd.ClientConfig session userd.Session sessionCancel context.CancelFunc sessionContext context.Context @@ -90,6 +93,13 @@ type service struct { self userd.Service } +func (s *service) ClientConfig() (clientcmd.ClientConfig, error) { + if s.clientConfig == nil { + return nil, errors.New("user daemon has no client config") + } + return s.clientConfig, nil +} + func NewService(ctx context.Context, _ *dgroup.Group, cfg client.Config, srv *grpc.Server) (userd.Service, error) { s := &service{ srv: srv, @@ -103,6 +113,7 @@ func NewService(ctx context.Context, _ *dgroup.Group, cfg client.Config, srv *gr if srv != nil { // The podd daemon never registers the gRPC servers rpc.RegisterConnectorServer(srv, s) + authGrpc.RegisterAuthenticatorServer(srv, s) rpc.RegisterManagerProxyServer(srv, s.managerProxy) } else { s.rootSessionInProc = true @@ -261,6 +272,7 @@ func (s *service) startSession(parentCtx context.Context, cr userd.ConnectReques ErrorCategory: int32(errcat.GetCategory(err)), } } + s.clientConfig = config.ClientConfig ctx, cancel := context.WithCancel(ctx) ctx = userd.WithService(ctx, s.self) @@ -299,6 +311,7 @@ func (s *service) startSession(parentCtx context.Context, cr userd.ConnectReques defer func() { s.sessionLock.Lock() s.self.SetManagerClient(nil) + s.clientConfig = nil s.session = nil s.sessionCancel = nil s.sessionLock.Unlock() @@ -417,9 +430,6 @@ func run(cmd *cobra.Command, _ []string) error { return err } daemonAddress = grpcListener.Addr().(*net.TCPAddr) - defer func() { - _ = grpcListener.Close() - }() } else { socketPath := socket.UserDaemonPath(c) dlog.Infof(c, "Starting socket listener for %s", socketPath) @@ -451,21 +461,20 @@ func run(cmd *cobra.Command, _ []string) error { // Start services from within a group routine so that it gets proper cancellation // when the group is cancelled. siCh := make(chan userd.Service) - g.Go("service", func(c context.Context) error { + g.Go("serve-grpc", func(c context.Context) error { var opts []grpc.ServerOption if mz := cfg.Grpc().MaxReceiveSize(); mz > 0 { opts = append(opts, grpc.MaxRecvMsgSize(int(mz))) } - si, err := userd.GetNewServiceFunc(c)(c, g, cfg, grpc.NewServer(opts...)) + svc := server.New(c, opts...) + si, err := userd.GetNewServiceFunc(c)(c, g, cfg, svc) if err != nil { close(siCh) return err } siCh <- si close(siCh) - - <-c.Done() // wait for context cancellation - return nil + return server.Serve(c, svc, grpcListener) }) si, ok := <-siCh @@ -493,20 +502,6 @@ func run(cmd *cobra.Command, _ []string) error { }) } - g.Go("server-grpc", func(c context.Context) (err error) { - sc := &dhttp.ServerConfig{Handler: s.srv} - dlog.Info(c, "gRPC server started") - if err = sc.Serve(c, grpcListener); err != nil && c.Err() != nil { - err = nil // Normal shutdown - } - if err != nil { - dlog.Errorf(c, "gRPC server ended with: %v", err) - } else { - dlog.Debug(c, "gRPC server ended") - } - return err - }) - g.Go("config-reload", s.configReload) g.Go(sessionName, func(c context.Context) error { return s.ManageSessions(c) diff --git a/pkg/client/userd/session.go b/pkg/client/userd/session.go index 3cf170ea8a..b3c621a003 100644 --- a/pkg/client/userd/session.go +++ b/pkg/client/userd/session.go @@ -8,6 +8,7 @@ import ( core "k8s.io/api/core/v1" typed "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "github.com/datawire/dlib/dgroup" "github.com/telepresenceio/telepresence/rpc/v2/common" @@ -38,6 +39,7 @@ type InterceptInfo interface { type KubeConfig interface { GetContext() string GetRestConfig() *rest.Config + GetClientConfig() clientcmd.ClientConfig } type NamespaceListener func(context.Context) diff --git a/pkg/client/userd/trafficmgr/session.go b/pkg/client/userd/trafficmgr/session.go index ec7a374411..78a07073f6 100644 --- a/pkg/client/userd/trafficmgr/session.go +++ b/pkg/client/userd/trafficmgr/session.go @@ -34,14 +34,12 @@ import ( "github.com/datawire/dlib/dgroup" "github.com/datawire/dlib/dlog" "github.com/datawire/dlib/dtime" - "github.com/telepresenceio/telepresence/rpc/v2/authenticator" "github.com/telepresenceio/telepresence/rpc/v2/common" "github.com/telepresenceio/telepresence/rpc/v2/connector" rpc "github.com/telepresenceio/telepresence/rpc/v2/connector" rootdRpc "github.com/telepresenceio/telepresence/rpc/v2/daemon" "github.com/telepresenceio/telepresence/rpc/v2/manager" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" - authGrpc "github.com/telepresenceio/telepresence/v2/pkg/authenticator/grpc" "github.com/telepresenceio/telepresence/v2/pkg/authenticator/patcher" "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/client/cli/daemon" @@ -277,11 +275,7 @@ func NewSession( // Root daemon needs this to authenticate with the cluster. Potential exec configurations in the kubeconfig // must be executed by the user, not by root. konfig, err := patcher.CreateExternalKubeConfig(ctx, config.ClientConfig, cluster.Context, func([]string) (string, string, error) { - s := userd.GetService(ctx) - if _, ok := s.Server().GetServiceInfo()[authenticator.Authenticator_ServiceDesc.ServiceName]; !ok { - authGrpc.RegisterAuthenticatorServer(s.Server(), config.ClientConfig) - } - return client.GetExe(ctx), s.ListenerAddress(ctx), nil + return client.GetExe(ctx), userd.GetService(ctx).ListenerAddress(ctx), nil }, nil) if err != nil { return ctx, nil, connectError(rpc.ConnectInfo_DAEMON_FAILED, err) diff --git a/pkg/grpc/server/server.go b/pkg/grpc/server/server.go new file mode 100644 index 0000000000..994df9adfd --- /dev/null +++ b/pkg/grpc/server/server.go @@ -0,0 +1,97 @@ +package server + +import ( + "context" + "net" + + "google.golang.org/grpc" + + "github.com/datawire/dlib/dcontext" + "github.com/datawire/dlib/dlog" +) + +type mergedCtx struct { + context.Context + valCtx context.Context +} + +func (m *mergedCtx) Value(i any) any { + if v := m.valCtx.Value(i); v != nil { + return v + } + return m.Context.Value(i) +} + +type mergedStream struct { + grpc.ServerStream + valCtx context.Context +} + +func (s *mergedStream) Context() context.Context { + return &mergedCtx{Context: s.ServerStream.Context(), valCtx: s.valCtx} +} + +// New creates a gRPC server which has no service registered and has not started to accept requests yet. Values +// in the provided context will be included in the context passed to both unary and stream calls. +func New(valCtx context.Context, options ...grpc.ServerOption) *grpc.Server { + unaryInterceptor := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return handler(&mergedCtx{Context: ctx, valCtx: valCtx}, req) + } + streamInterceptor := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return handler(srv, &mergedStream{ + ServerStream: ss, + valCtx: valCtx, + }) + } + options = append(options, grpc.UnaryInterceptor(unaryInterceptor), grpc.StreamInterceptor(streamInterceptor)) + return grpc.NewServer(options...) +} + +// Serve accepts incoming connections on the listener lis, creating a new ServerTransport and service goroutine for each. +// The service goroutines read gRPC requests and then call the registered handlers to reply to them. Serve returns when +// lis.Accept fails with fatal errors. +// +// Serve waits until ctx.Done is closed. The svc.GracefulStop function will be called if the context has soft-cancel +// enabled. The svc.Stop function will be called if no soft-cancel is enabled or when the GracefulStop doesn't finish +// until the hard context is done. +func Serve(ctx context.Context, svc *grpc.Server, lis net.Listener) error { + dlog.Debug(ctx, "gRPC server started") + go Wait(ctx, svc) + err := svc.Serve(lis) + if err != nil { + dlog.Errorf(ctx, "gRPC server ended with error: %v", err) + } else { + dlog.Debug(ctx, "gRPC server ended") + } + return err +} + +// Wait waits until the given contexts Done channel is closed. The server's GracefulStop function will be called +// if the context has soft-cancel enabled. The server's Stop function will be called if no soft-cancel is enabled or +// when the GracefulStop doesn't finish until the Done channel of the hard context closed. +func Wait(ctx context.Context, svc *grpc.Server) { + hardCtx := dcontext.HardContext(ctx) + if hardCtx != ctx { + <-ctx.Done() + dead := make(chan struct{}) + go func() { + dlog.Debug(ctx, "Initiating soft shutdown") + svc.GracefulStop() + close(dead) + dlog.Debug(ctx, "Soft shutdown complete") + }() + select { + case <-dead: + // GracefulStop did the job. + case <-hardCtx.Done(): + dlog.Debug(ctx, "Initiating hard shutdown") + svc.Stop() + dlog.Debug(ctx, "Hard shutdown complete") + } + } else { + <-ctx.Done() + dlog.Debug(ctx, "Initiating hard shutdown") + svc.Stop() + dlog.Debug(ctx, "Hard shutdown complete") + } +} diff --git a/pkg/k8sapi/clientconfigprovider.go b/pkg/k8sapi/clientconfigprovider.go new file mode 100644 index 0000000000..574a6294a8 --- /dev/null +++ b/pkg/k8sapi/clientconfigprovider.go @@ -0,0 +1,7 @@ +package k8sapi + +import "k8s.io/client-go/tools/clientcmd" + +type ClientConfigProvider interface { + ClientConfig() (clientcmd.ClientConfig, error) +} From f65c9e9cbcecead4e91cd3766262bf930c76ff2e Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 28 Jan 2025 18:34:12 +0100 Subject: [PATCH 22/61] General overhaul of client daemon quit logic. Signed-off-by: Thomas Hallgren --- pkg/authenticator/authenticator_test.go | 7 ++- pkg/client/agentpf/clients.go | 1 + pkg/client/rootd/service.go | 56 +++++++---------------- pkg/client/userd/daemon/grpc.go | 7 ++- pkg/client/userd/daemon/service.go | 61 ++++++++++--------------- pkg/client/userd/service.go | 2 +- pkg/tunnel/dialer.go | 18 ++++++-- 7 files changed, 66 insertions(+), 86 deletions(-) diff --git a/pkg/authenticator/authenticator_test.go b/pkg/authenticator/authenticator_test.go index a115dac6c6..9615d5e801 100644 --- a/pkg/authenticator/authenticator_test.go +++ b/pkg/authenticator/authenticator_test.go @@ -8,6 +8,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "k8s.io/client-go/tools/clientcmd" clientcmd_api "k8s.io/client-go/tools/clientcmd/api" mock_authenticator "github.com/telepresenceio/telepresence/v2/pkg/authenticator/mocks" @@ -36,6 +37,10 @@ type SuiteService struct { service *Service } +func (s *SuiteService) ClientConfig() (clientcmd.ClientConfig, error) { + return s.kubeClientConfig, nil +} + func (s *SuiteService) SetupTest() { s.ctrl = gomock.NewController(s.T()) @@ -43,7 +48,7 @@ func (s *SuiteService) SetupTest() { s.execCredentialsResolver = mock_authenticator.NewMockExecCredentialsResolver(s.ctrl) s.service = &Service{ - kubeClientConfig: s.kubeClientConfig, + clientConfigProvider: s, execCredentialsResolver: s.execCredentialsResolver, } } diff --git a/pkg/client/agentpf/clients.go b/pkg/client/agentpf/clients.go index 19cf59e9b4..4b507b6ab9 100644 --- a/pkg/client/agentpf/clients.go +++ b/pkg/client/agentpf/clients.go @@ -232,6 +232,7 @@ func (ac *client) startDialWatcherReady(ctx context.Context) error { if err != nil { dlog.Error(ctx, err) } + ac.cancel() }() return nil } diff --git a/pkg/client/rootd/service.go b/pkg/client/rootd/service.go index cc5a688a97..44b0453f78 100644 --- a/pkg/client/rootd/service.go +++ b/pkg/client/rootd/service.go @@ -147,16 +147,7 @@ func (s *Service) Status(context.Context, *emptypb.Empty) (*rpc.DaemonStatus, er func (s *Service) Quit(ctx context.Context, _ *emptypb.Empty) (*emptypb.Empty, error) { dlog.Debug(ctx, "Received gRPC Quit") - if !s.sessionLock.TryRLock() { - // A running session is blocking with a write-lock. Give it some time to quit, then kill it - time.Sleep(2 * time.Second) - if !s.sessionLock.TryRLock() { - s.quit() - return &emptypb.Empty{}, nil - } - } - defer s.sessionLock.RUnlock() - s.cancelSessionReadLocked() + s.cancelSession() s.quit() return &emptypb.Empty{}, nil } @@ -233,25 +224,15 @@ func (s *Service) WaitForNetwork(ctx context.Context, e *emptypb.Empty) (*emptyp return &emptypb.Empty{}, err } -func (s *Service) cancelSessionReadLocked() { - if s.sessionCancel != nil { - s.sessionCancel() - } -} - func (s *Service) cancelSession() { - if !atomic.CompareAndSwapInt32(&s.sessionQuitting, 0, 1) { - return + if atomic.CompareAndSwapInt32(&s.sessionQuitting, 0, 1) { + if s.sessionCancel != nil { + s.sessionCancel() + } + s.session = nil + s.sessionCancel = nil + atomic.StoreInt32(&s.sessionQuitting, 0) } - s.sessionLock.RLock() - s.cancelSessionReadLocked() - s.sessionLock.RUnlock() - - s.sessionLock.Lock() - s.session = nil - s.sessionCancel = nil - atomic.StoreInt32(&s.sessionQuitting, 0) - s.sessionLock.Unlock() } func (s *Service) WithSession(f func(context.Context, *Session) error) error { @@ -360,7 +341,12 @@ func (s *Service) startSession(parentCtx context.Context, oi *rpc.NetworkConfig, s.sessionContext = ctx s.sessionCancel = func() { cancel() - <-session.Done() + wCtx, wCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer wCancel() + select { + case <-session.Done(): + case <-wCtx.Done(): + } } _ = client.ReloadDaemonLogLevel(ctx, true) reply.status.OutboundConfig = s.session.getNetworkConfig(ctx) @@ -373,15 +359,7 @@ func (s *Service) startSession(parentCtx context.Context, oi *rpc.NetworkConfig, wg.Add(1) go func() { defer func() { - if r := recover(); r != nil { - s.sessionLock.TryLock() - s.sessionLock.Unlock() - dlog.Errorf(ctx, "%+v", derror.PanicToError(r)) - } - s.sessionLock.Lock() - s.session = nil - s.sessionCancel = nil - s.sessionLock.Unlock() + s.cancelSession() _ = client.ReloadDaemonLogLevel(parentCtx, true) wg.Done() }() @@ -394,7 +372,7 @@ func (s *Service) startSession(parentCtx context.Context, oi *rpc.NetworkConfig, case err := <-initErrCh: if err != nil { reply.err = err - s.cancelSessionReadLocked() + s.cancelSession() } } return reply @@ -481,7 +459,7 @@ func run(cmd *cobra.Command, args []string) error { vif.InitLogger(c) g := dgroup.NewGroup(c, dgroup.GroupConfig{ - SoftShutdownTimeout: 2 * time.Second, + SoftShutdownTimeout: 5 * time.Second, EnableSignalHandling: true, ShutdownOnNonError: true, }) diff --git a/pkg/client/userd/daemon/grpc.go b/pkg/client/userd/daemon/grpc.go index 48a1ebdd3d..cb728ac835 100644 --- a/pkg/client/userd/daemon/grpc.go +++ b/pkg/client/userd/daemon/grpc.go @@ -438,11 +438,10 @@ func (s *service) SetLogLevel(ctx context.Context, request *rpc.LogLevelRequest) func (s *service) Quit(ctx context.Context, ex *empty.Empty) (*empty.Empty, error) { s.LogCall(ctx, "Quit", func(c context.Context) { - s.sessionLock.RLock() - defer s.sessionLock.RUnlock() - s.cancelSessionReadLocked() + s.cancelSession() s.quit() - _ = s.withRootDaemon(ctx, func(ctx context.Context, rd daemon.DaemonClient) error { + _ = s.withRootDaemon(context.WithoutCancel(ctx), func(ctx context.Context, rd daemon.DaemonClient) error { + dlog.Debug(ctx, "Telling root daemon to Quit") _, err := rd.Quit(ctx, ex) return err }) diff --git a/pkg/client/userd/daemon/service.go b/pkg/client/userd/daemon/service.go index 895b328524..a23a919080 100644 --- a/pkg/client/userd/daemon/service.go +++ b/pkg/client/userd/daemon/service.go @@ -65,11 +65,6 @@ type service struct { // The quit function that quits the server. quit func() - // quitDisable will temporarily disable the quit function. This is used when there's a desire - // to cancel the session without cancelling the process although the simplified session management - // is in effect (rootSessionInProc == true). - quitDisable bool - clientConfig clientcmd.ClientConfig session userd.Session sessionCancel context.CancelFunc @@ -100,7 +95,7 @@ func (s *service) ClientConfig() (clientcmd.ClientConfig, error) { return s.clientConfig, nil } -func NewService(ctx context.Context, _ *dgroup.Group, cfg client.Config, srv *grpc.Server) (userd.Service, error) { +func NewService(ctx context.Context, cancel context.CancelFunc, _ *dgroup.Group, cfg client.Config, srv *grpc.Server) (userd.Service, error) { s := &service{ srv: srv, connectRequest: make(chan userd.ConnectRequest), @@ -108,6 +103,7 @@ func NewService(ctx context.Context, _ *dgroup.Group, cfg client.Config, srv *gr managerProxy: &mgrProxy{}, timedLogLevel: log.NewTimedLevel(cfg.LogLevels().UserDaemon.String(), log.SetLevel), fuseFtpMgr: remotefs.NewFuseFTPManager(), + quit: cancel, } s.self = s if srv != nil { @@ -117,7 +113,6 @@ func NewService(ctx context.Context, _ *dgroup.Group, cfg client.Config, srv *gr rpc.RegisterManagerProxyServer(srv, s.managerProxy) } else { s.rootSessionInProc = true - s.quit = func() {} } return s, nil } @@ -222,12 +217,6 @@ func (s *service) configReload(c context.Context) error { // a session and writes a reply to the connectErrCh. The session is then started if it was // successfully created. func (s *service) ManageSessions(c context.Context) error { - c, cancel := context.WithCancel(c) - s.quit = func() { - if !s.quitDisable { - cancel() - } - } wg := sync.WaitGroup{} defer wg.Wait() @@ -363,30 +352,25 @@ func runAliveAndCancellation(ctx context.Context, cancel context.CancelFunc, dae } } -func (s *service) cancelSessionReadLocked() { - if s.sessionCancel != nil { - if err := s.session.ClearIngestsAndIntercepts(s.sessionContext); err != nil { - dlog.Errorf(s.sessionContext, "failed to clear intercepts: %v", err) - } - s.sessionCancel() - } -} - func (s *service) cancelSession() { - if !atomic.CompareAndSwapInt32(&s.sessionQuitting, 0, 1) { - return + if atomic.CompareAndSwapInt32(&s.sessionQuitting, 0, 1) { + if s.sessionCancel != nil { + // We use a TryRLock here because the session-lock will be held during + // session initialization, and we might well receive a quit-call during + // that time (the initialization may take a long time if there are + // problems connecting to the cluster). + if s.sessionLock.TryRLock() { + if err := s.session.ClearIngestsAndIntercepts(s.sessionContext); err != nil { + dlog.Errorf(s.sessionContext, "failed to clear intercepts: %v", err) + } + s.sessionLock.RUnlock() + } + s.sessionCancel() + } + s.session = nil + s.sessionCancel = nil + atomic.StoreInt32(&s.sessionQuitting, 0) } - s.sessionLock.RLock() - s.cancelSessionReadLocked() - s.sessionLock.RUnlock() - - // We have to cancel the session before we can acquire this write-lock, because we need any long-running RPCs - // that may be holding the RLock to die. - s.sessionLock.Lock() - s.session = nil - s.sessionCancel = nil - atomic.StoreInt32(&s.sessionQuitting, 0) - s.sessionLock.Unlock() } // run is the main function when executing as the connector. @@ -422,6 +406,7 @@ func run(cmd *cobra.Command, _ []string) error { if err != nil { return err } + rootSessionInProc, _ := flags.GetBool(embedNetworkFlag) var daemonAddress *net.TCPAddr if addr, _ := flags.GetString(addressFlag); addr != "" { @@ -462,12 +447,16 @@ func run(cmd *cobra.Command, _ []string) error { // when the group is cancelled. siCh := make(chan userd.Service) g.Go("serve-grpc", func(c context.Context) error { + // svcCancel is what a `quit -s` call will cancel. The Group provides soft cancellation to it. + // which will result in a graceful termination of the grpc server. + c, svcCancel := context.WithCancel(c) + var opts []grpc.ServerOption if mz := cfg.Grpc().MaxReceiveSize(); mz > 0 { opts = append(opts, grpc.MaxRecvMsgSize(int(mz))) } svc := server.New(c, opts...) - si, err := userd.GetNewServiceFunc(c)(c, g, cfg, svc) + si, err := userd.GetNewServiceFunc(c)(c, svcCancel, g, cfg, svc) if err != nil { close(siCh) return err diff --git a/pkg/client/userd/service.go b/pkg/client/userd/service.go index 33652833c5..046a8e3e8a 100644 --- a/pkg/client/userd/service.go +++ b/pkg/client/userd/service.go @@ -44,7 +44,7 @@ type Service interface { ManageSessions(context.Context) error } -type NewServiceFunc func(context.Context, *dgroup.Group, client.Config, *grpc.Server) (Service, error) +type NewServiceFunc func(context.Context, context.CancelFunc, *dgroup.Group, client.Config, *grpc.Server) (Service, error) type newServiceKey struct{} diff --git a/pkg/tunnel/dialer.go b/pkg/tunnel/dialer.go index 6ad1357edf..81b9cf3ebc 100644 --- a/pkg/tunnel/dialer.go +++ b/pkg/tunnel/dialer.go @@ -340,13 +340,21 @@ func DialWaitLoop( defer cancel() for ctx.Err() == nil { dr, err := dialStream.Recv() - if err != nil { - if ctx.Err() == nil && !(errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) || status.Code(err) == codes.NotFound) { - return fmt.Errorf("dial request stream recv: %w", err) - } + if err == nil { + go dialRespond(ctx, tunnelProvider, dr, sessionID) + continue + } + if ctx.Err() != nil { + return nil + } + if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) || status.Code(err) == codes.NotFound { + return nil + } + switch status.Code(err) { + case codes.NotFound, codes.Unavailable: return nil } - go dialRespond(ctx, tunnelProvider, dr, sessionID) + return fmt.Errorf("dial request stream recv: %w", err) } return nil } From c81fed0ce23c5681c37fc366ca87a4a8e39663d1 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 28 Jan 2025 18:34:56 +0100 Subject: [PATCH 23/61] Remove some excessive debug logging. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/manager/mutator/agent_injector.go | 10 +++++----- cmd/traffic/cmd/manager/service.go | 3 --- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/traffic/cmd/manager/mutator/agent_injector.go b/cmd/traffic/cmd/manager/mutator/agent_injector.go index fe3525e65c..2aaa44e806 100644 --- a/cmd/traffic/cmd/manager/mutator/agent_injector.go +++ b/cmd/traffic/cmd/manager/mutator/agent_injector.go @@ -200,19 +200,19 @@ func (a *agentInjector) Inject(ctx context.Context, req *admission.AdmissionRequ // Create patch operations to add the traffic-agent sidecar if len(patches) > 0 { - dlog.Infof(ctx, "Injecting %d patches into pod %s.%s", len(patches), pod.Name, pod.Namespace) - if dlog.MaxLogLevel(ctx) >= dlog.LogLevelDebug { + dlog.Debugf(ctx, "Injecting %d patches into pod %s.%s", len(patches), pod.Name, pod.Namespace) + if dlog.MaxLogLevel(ctx) >= dlog.LogLevelTrace { cns := strings.Builder{} for i, cn := range pod.Spec.Containers { cns.WriteString(fmt.Sprintf("%d %s\n", i, cn.Name)) } - dlog.Debugf(ctx, "Containers \n%s", cns.String()) + dlog.Tracef(ctx, "Containers \n%s", cns.String()) if pj, err := json.Marshal(patches, jsontext.WithIndent(" ")); err == nil { - dlog.Debugf(ctx, "\n%s", string(pj)) + dlog.Tracef(ctx, "\n%s", string(pj)) } } } else { - dlog.Infof(ctx, "Pod %s.%s was left untouched", pod.Name, pod.Namespace) + dlog.Debugf(ctx, "Pod %s.%s was left untouched", pod.Name, pod.Namespace) } return patches, nil } diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index e3f46937d4..9cd63a86e8 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -486,14 +486,11 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W rpc.InterceptDispositionType_ACTIVE, rpc.InterceptDispositionType_AGENT_ERROR: // agent-owned state: include the intercept - dlog.Debugf(ctx, "Intercept %s.%s valid. Disposition: %s", info.Spec.Agent, info.Spec.Namespace, info.Disposition) return true case rpc.InterceptDispositionType_REMOVED: - dlog.Debugf(ctx, "Intercept %s.%s valid but removed", info.Spec.Agent, info.Spec.Namespace) return true default: // otherwise: don't return this intercept - dlog.Debugf(ctx, "Intercept %s.%s is not in agent-owned state. Disposition: %s", info.Spec.Agent, info.Spec.Namespace, info.Disposition) return false } } From 11384f5352ba4d28a9244f10b8a7584609cbce2e Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 28 Jan 2025 18:35:34 +0100 Subject: [PATCH 24/61] Clean up traffic-manager code dealing with session termination. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/manager/state/state.go | 67 +++++++++++++---------- integration_test/itest/traffic_manager.go | 2 +- pkg/client/rootd/service.go | 5 +- 3 files changed, 41 insertions(+), 33 deletions(-) diff --git a/cmd/traffic/cmd/manager/state/state.go b/cmd/traffic/cmd/manager/state/state.go index cc1d23c680..d0486b30d4 100644 --- a/cmd/traffic/cmd/manager/state/state.go +++ b/cmd/traffic/cmd/manager/state/state.go @@ -300,12 +300,10 @@ func (s *state) RemoveSession(ctx context.Context, sessionID string) { // kill the session defer sess.Cancel() - s.gcSessionIntercepts(ctx, sessionID) - - agent, isAgent := s.agents.LoadAndDelete(sessionID) - if isAgent { - mutator.GetMap(s.backgroundCtx).Inactivate(agent.PodName) + if agent, isAgent := s.agents.LoadAndDelete(sessionID); isAgent { + s.consolidateAgentSessionIntercepts(ctx, agent) } else if client, isClient := s.clients.LoadAndDelete(sessionID); isClient { + s.gcClientSessionIntercepts(ctx, sessionID, client) scm := sess.(*clientSessionState).consumptionMetrics atomic.AddUint64(&s.tunnelIngressCounter, scm.FromClientBytes.GetValue()) atomic.AddUint64(&s.tunnelEgressCounter, scm.ToClientBytes.GetValue()) @@ -315,39 +313,48 @@ func (s *state) RemoveSession(ctx context.Context, sessionID string) { }) } -func (s *state) gcSessionIntercepts(ctx context.Context, sessionID string) { - // GC any intercepts that relied on this session; prune any intercepts that - // 1. Don't have a client session (intercept.ClientSession.SessionId) - // 2. Don't have any agents (agent.PodIp == intercept.PodIp) - // Alternatively, if the intercept is still live but has been switched over to a different agent, send it back to - // WAITING state +func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *rpc.AgentInfo) { + dlog.Debugf(ctx, "Consolidating intercepts after removal of agent %s", agent.PodName) + mutator.GetMap(s.backgroundCtx).Inactivate(agent.PodName) s.intercepts.Range(func(interceptID string, intercept *rpc.InterceptInfo) bool { - if intercept.Disposition == rpc.InterceptDispositionType_REMOVED { + if intercept.Disposition == rpc.InterceptDispositionType_REMOVED || agent.PodIp != intercept.PodIp { + // Not of interest. Continue iteration. return true } - if intercept.ClientSession.SessionId == sessionID { - // Client went away: - // Delete it. - if client := s.GetClient(sessionID); client != nil { - wl := strings.SplitN(interceptID, ":", 2)[1] - s.allInterceptsFinalizerCall(client, &wl) - } - s.self.RemoveIntercept(ctx, interceptID) - } else if errCode, errMsg := s.checkAgentsForIntercept(intercept); errCode != 0 { - // Refcount went to zero: - // Tell the client, so that the client can tell us to delete it. + + if errCode, errMsg := s.checkAgentsForIntercept(intercept); errCode != rpc.InterceptDispositionType_UNSPECIFIED { + // No agents matching this intercept are available, so the intercept is now dormant or in error. + dlog.Debugf(ctx, "Intercept %q no longer has available agents. Setting it disposition to %s", interceptID, errCode) s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { intercept.Disposition = errCode intercept.Message = errMsg }) } else { - if agent := s.GetAgent(sessionID); agent != nil && agent.PodIp == intercept.PodIp { - // The agent whose podIP was stored by the intercept is dead, but it's not the last agent - // Send it back to waiting so that one of the other agents can pick it up and set their own podIP - s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { - intercept.Disposition = rpc.InterceptDispositionType_WAITING - }) - } + // The agent is about to die, but apparently more agents are present. Let some other agent pick it up then. + dlog.Debugf(ctx, "Intercept %q lost its agent pod %s. Setting it disposition to WAITING", interceptID, agent.PodName) + s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { + intercept.PodIp = "" + intercept.PodName = "" + intercept.Disposition = rpc.InterceptDispositionType_WAITING + }) + } + // Only one agent with this IP. Break iteration. + return false + }) +} + +func (s *state) gcClientSessionIntercepts(ctx context.Context, sessionID string, client *rpc.ClientInfo) { + // GC all intercepts for the client session (intercept.ClientSession.SessionId) + s.intercepts.Range(func(interceptID string, intercept *rpc.InterceptInfo) bool { + if intercept.Disposition == rpc.InterceptDispositionType_REMOVED { + return true + } + if intercept.ClientSession.SessionId == sessionID { + // Client went away: + // Delete it. + wl := strings.SplitN(interceptID, ":", 2)[1] + s.allInterceptsFinalizerCall(client, &wl) + s.self.RemoveIntercept(ctx, interceptID) } return true }) diff --git a/integration_test/itest/traffic_manager.go b/integration_test/itest/traffic_manager.go index d79af2af56..a7955549f5 100644 --- a/integration_test/itest/traffic_manager.go +++ b/integration_test/itest/traffic_manager.go @@ -183,7 +183,7 @@ func (th *trafficManager) DoWithSession(ctx context.Context, cr *rpc.ConnectRequ ShutdownOnNonError: true, }) - srv, err := userd.GetNewServiceFunc(ctx)(ctx, g, client.GetConfig(ctx), grpc.NewServer()) + srv, err := userd.GetNewServiceFunc(ctx)(ctx, cancel, g, client.GetConfig(ctx), grpc.NewServer()) if err != nil { return err } diff --git a/pkg/client/rootd/service.go b/pkg/client/rootd/service.go index 44b0453f78..6617830307 100644 --- a/pkg/client/rootd/service.go +++ b/pkg/client/rootd/service.go @@ -286,7 +286,6 @@ func (s *Service) manageSessions(c context.Context) error { // terminates this function, it terminates the whole process. wg := sync.WaitGroup{} defer wg.Wait() - c, s.quit = context.WithCancel(c) for { // Wait for a connection request @@ -378,10 +377,11 @@ func (s *Service) startSession(parentCtx context.Context, oi *rpc.NetworkConfig, return reply } -func (s *Service) serveGrpc(c context.Context, l net.Listener) error { +func (s *Service) serveGrpc(c context.Context, l net.Listener) (err error) { defer func() { // Error recovery. if perr := derror.PanicToError(recover()); perr != nil { + err = perr dlog.Errorf(c, "%+v", perr) } }() @@ -391,6 +391,7 @@ func (s *Service) serveGrpc(c context.Context, l net.Listener) error { if mz := cfg.Grpc().MaxReceiveSize(); mz > 0 { opts = append(opts, grpc.MaxRecvMsgSize(int(mz))) } + c, s.quit = context.WithCancel(c) svc := server.New(c, opts...) rpc.RegisterDaemonServer(svc, s) return server.Serve(c, svc, l) From 6a0ce93868881b126672e2c018124619091eb967 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 28 Jan 2025 18:37:18 +0100 Subject: [PATCH 25/61] Fix some issues with pod-scaling integration test. Signed-off-by: Thomas Hallgren --- integration_test/itest/cluster.go | 21 +++++++++++++-------- integration_test/podscaling_test.go | 11 ++++++++--- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/integration_test/itest/cluster.go b/integration_test/itest/cluster.go index 9a0993ff5f..ca21c7a66c 100644 --- a/integration_test/itest/cluster.go +++ b/integration_test/itest/cluster.go @@ -510,17 +510,22 @@ func (s *cluster) CapturePodLogs(ctx context.Context, app, container, ns string) } present := struct{}{} - // Use another logger to avoid errors due to logs arriving after the tests complete. - ctx = dlog.WithLogger(ctx, dlog.WrapLogrus(logrus.StandardLogger())) - pod := pods[0] - key := pod - if container != "" { - key += "/" + container + var pod string + for i, key := range pods { + if container != "" { + key += "/" + container + } + if _, ok := s.logCapturingPods.LoadOrStore(key, present); !ok { + pod = pods[i] + break + } } - if _, ok := s.logCapturingPods.LoadOrStore(key, present); ok { - return "" + if pod == "" { + return "" // All pods already captured } + // Use another logger to avoid errors due to logs arriving after the tests complete. + ctx = dlog.WithLogger(ctx, dlog.WrapLogrus(logrus.StandardLogger())) logFile, err := os.Create( filepath.Join(filelocation.AppUserLogDir(ctx), fmt.Sprintf("%s-%s.log", dtime.Now().Format("20060102T150405"), pod))) if err != nil { diff --git a/integration_test/podscaling_test.go b/integration_test/podscaling_test.go index fa43427805..528a13369e 100644 --- a/integration_test/podscaling_test.go +++ b/integration_test/podscaling_test.go @@ -97,7 +97,7 @@ func (s *interceptMountSuite) Test_StopInterceptedPodOfMany() { assert.Eventually( func() bool { return len(s.runningPods(ctx)) == 1 - }, 15*time.Second, time.Second) + }, time.Minute, time.Second) s.CapturePodLogs(ctx, s.ServiceName(), "traffic-agent", s.AppNamespace()) }() @@ -118,7 +118,7 @@ func (s *interceptMountSuite) Test_StopInterceptedPodOfMany() { } } return len(pods) == 2 - }, 15*time.Second, time.Second) + }, time.Minute, time.Second) s.CapturePodLogs(ctx, s.ServiceName(), "traffic-agent", s.AppNamespace()) // Verify that intercept is still active @@ -136,6 +136,7 @@ func (s *interceptMountSuite) Test_StopInterceptedPodOfMany() { }, 15*time.Second, time.Second) // Verify response from intercepting client + expect := s.ServiceName() + " from intercept at /" require.Eventually(func() bool { hc := http.Client{Timeout: time.Second} resp, err := hc.Get("http://" + s.ServiceName()) @@ -147,7 +148,11 @@ func (s *interceptMountSuite) Test_StopInterceptedPodOfMany() { if err != nil { return false } - return s.ServiceName()+" from intercept at /" == string(body) + if expect == string(body) { + return true + } + dlog.Infof(ctx, "%s != %s", expect, string(body)) + return false }, 30*time.Second, time.Second) // Verify that volume mount is restored From ef9fd5df11307493a46cc17ee4fff3d3c50942ae Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 28 Jan 2025 18:37:41 +0100 Subject: [PATCH 26/61] Let kubeconfig_extension_test.go use proper client config. Signed-off-by: Thomas Hallgren --- integration_test/kubeconfig_extension_test.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/integration_test/kubeconfig_extension_test.go b/integration_test/kubeconfig_extension_test.go index 8458dc7c40..064f2ed8e0 100644 --- a/integration_test/kubeconfig_extension_test.go +++ b/integration_test/kubeconfig_extension_test.go @@ -421,12 +421,22 @@ func (s *notConnectedSuite) Test_DNSSuffixRules() { }) } ctx = itest.WithKubeConfigExtension(ctx, func(cluster *api.Cluster) map[string]any { - return map[string]any{ - "dns": map[string][]string{ - "exclude-suffixes": tt.excludeSuffixes, - "include-suffixes": tt.includeSuffixes, + m := map[string]any{ + "logLevels": map[string]string{ + "rootDaemon": "trace", }, } + if len(tt.excludeSuffixes)+len(tt.includeSuffixes) > 0 { + dns := map[string]any{} + if len(tt.excludeSuffixes) > 0 { + dns["excludeSuffixes"] = tt.excludeSuffixes + } + if len(tt.includeSuffixes) > 0 { + dns["includeSuffixes"] = tt.includeSuffixes + } + m["dns"] = dns + } + return m }) require := s.Require() From 037d9e4ea617ca437e858dad52cc397fd1f5f8df Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 28 Jan 2025 18:38:18 +0100 Subject: [PATCH 27/61] Ensure that generic.goyaml can use fully qualified image name. Signed-off-by: Thomas Hallgren --- integration_test/testdata/k8s/generic.goyaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/integration_test/testdata/k8s/generic.goyaml b/integration_test/testdata/k8s/generic.goyaml index 6a33f9c325..b1f1e705e1 100644 --- a/integration_test/testdata/k8s/generic.goyaml +++ b/integration_test/testdata/k8s/generic.goyaml @@ -48,7 +48,11 @@ spec: spec: containers: - name: backend - image: "{{ .Registry }}/{{ .Image }}" +{{- if .Registry }} + image: {{ .Registry }}/{{ .Image }} +{{- else }} + image: {{ .Image }} +{{- end}} ports: {{- range .ContainerPorts }} - containerPort: {{ .Number }} From 99dd55edf14d485b33a909f1b4ba7293e594c3c3 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 28 Jan 2025 23:43:43 +0100 Subject: [PATCH 28/61] We must use podIP to identify agents. StatefulSet will reuse pod names. Signed-off-by: Thomas Hallgren --- .../cmd/manager/mutator/agent_injector.go | 4 ++-- cmd/traffic/cmd/manager/mutator/watcher.go | 15 ++++++--------- cmd/traffic/cmd/manager/service.go | 10 +++++----- cmd/traffic/cmd/manager/state/intercept.go | 6 +++--- cmd/traffic/cmd/manager/state/state.go | 16 ++++++++-------- .../cmd/manager/state/workload_info_watcher.go | 4 ++-- pkg/agentmap/discorvery.go | 8 ++++---- 7 files changed, 30 insertions(+), 33 deletions(-) diff --git a/cmd/traffic/cmd/manager/mutator/agent_injector.go b/cmd/traffic/cmd/manager/mutator/agent_injector.go index 2aaa44e806..3c30d180c8 100644 --- a/cmd/traffic/cmd/manager/mutator/agent_injector.go +++ b/cmd/traffic/cmd/manager/mutator/agent_injector.go @@ -110,7 +110,7 @@ func (a *agentInjector) Inject(ctx context.Context, req *admission.AdmissionRequ } if isDelete { - a.agentConfigs.Inactivate(pod.Name) + a.agentConfigs.Inactivate(pod.Status.PodIP) return nil, nil } @@ -414,7 +414,7 @@ func addAgentContainer( return patches, replaceAnnotations } - refPodName := pod.Name + "." + pod.Namespace + refPodName := pod.Name + "(" + pod.Status.PodIP + ")" for i := range pod.Spec.Containers { pcn := &pod.Spec.Containers[i] if pcn.Name == agentconfig.ContainerName { diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index c374e383ac..d0d77e0fe5 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -37,8 +37,8 @@ type Map interface { OnAdd(context.Context, k8sapi.Workload, agentconfig.SidecarExt) error OnDelete(context.Context, string, string) error DeleteMapsAndRolloutAll(context.Context) - IsInactive(podName string) bool - Inactivate(podName string) + IsInactive(podIP string) bool + Inactivate(podIP string) DeletePodsWithConfig(ctx context.Context, wl k8sapi.Workload) error DeletePodsWithConfigMismatch(ctx context.Context, scx agentconfig.SidecarExt) error DeleteAllPodsWithConfig(ctx context.Context, namespace string) error @@ -572,22 +572,22 @@ func (c *configWatcher) DeleteAllPodsWithConfig(ctx context.Context, namespace s func (c *configWatcher) DeleteIfMismatch(ctx context.Context, pod *core.Pod, cfgJSON string) error { if c.IsDeleted(pod.Name) { - dlog.Debugf(ctx, "Skipping pod %s because it is already deleted", pod.Name) + dlog.Tracef(ctx, "Skipping pod %s because it is already deleted", pod.Name) return nil } a := pod.ObjectMeta.Annotations if v, ok := a[agentconfig.ManualInjectAnnotation]; ok && v == "true" { - dlog.Debugf(ctx, "Skipping pod %s because it is managed manually", pod.Name) + dlog.Tracef(ctx, "Skipping pod %s because it is managed manually", pod.Name) return nil } if a[agentconfig.ConfigAnnotation] == cfgJSON { - dlog.Debugf(ctx, "Keeping pod %s because its config is still valid", pod.Name) + dlog.Tracef(ctx, "Keeping pod %s because its config is still valid", pod.Name) return nil } var err error c.inactivePods.Compute(pod.Name, func(v inactivation, loaded bool) (inactivation, bool) { if loaded && v.deleted { - dlog.Debugf(ctx, "Skipping pod %s because it was deleted by another thread", pod.Name) + dlog.Tracef(ctx, "Skipping pod %s because it was deleted by another thread", pod.Name) return v, false } dlog.Debugf(ctx, "Deleting pod %s because its config is no longer valid", pod.Name) @@ -667,12 +667,9 @@ func podList(ctx context.Context, kind, name, namespace string) ([]*core.Pod, er if !podIsRunning(pod) { continue } - dlog.Debugf(ctx, "getting owner workload for pod %s.%s", pod.Name, pod.Namespace) wl, err := agentmap.FindOwnerWorkload(ctx, k8sapi.Pod(pod), supportedKinds) if err == nil { - dlog.Debugf(ctx, "owner workload for pod %s.%s is %s %s", pod.Name, pod.Namespace, wl.GetKind(), wl.GetName()) if (kind == "" || wl.GetKind() == kind) && (name == "" || wl.GetName() == name) { - dlog.Debugf(ctx, "owner workload was a match. Pod %s.%s is of interest", pod.Name, pod.Namespace) podsOfInterest = append(podsOfInterest, pod) } } else { diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index 9cd63a86e8..cb7b7f2304 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -209,7 +209,7 @@ func (s *service) ArriveAsClient(ctx context.Context, client *rpc.ClientInfo) (* // ArriveAsAgent establishes a session between an agent and the Manager. func (s *service) ArriveAsAgent(ctx context.Context, agent *rpc.AgentInfo) (*rpc.SessionInfo, error) { - dlog.Debugf(ctx, "ArriveAsAgent %s called", agent.PodName) + dlog.Debugf(ctx, "ArriveAsAgent %s(%s) called", agent.PodName, agent.PodIp) if val := validateAgent(agent); val != "" { return nil, status.Error(codes.InvalidArgument, val) } @@ -433,7 +433,7 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(string, *rp names[i] = a.PodName + "." + a.Namespace i++ } - dlog.Debugf(ctx, "WatchAgentsNS sending update %v", names) + dlog.Tracef(ctx, "WatchAgentsNS sending update %v", names) } resp := &rpc.AgentInfoSnapshot{ Agents: agents, @@ -516,7 +516,7 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W dlog.Debugf(ctx, "WatchIntercepts session no longer active") return nil } - dlog.Debugf(ctx, "WatchIntercepts sending update") + dlog.Tracef(ctx, "WatchIntercepts sending update") intercepts := make([]*rpc.InterceptInfo, 0, len(snapshot)) for _, intercept := range snapshot { intercepts = append(intercepts, intercept) @@ -695,8 +695,8 @@ func (s *service) ReviewIntercept(ctx context.Context, rIReq *rpc.ReviewIntercep if intercept.Spec.Namespace != agent.Namespace || intercept.Spec.Agent != agent.Name { return } - if mutator.GetMap(ctx).IsInactive(agent.PodName) { - dlog.Debugf(ctx, "Pod %s.%s is blacklisted", agent.PodName, agent.Namespace) + if mutator.GetMap(ctx).IsInactive(agent.PodIp) { + dlog.Debugf(ctx, "Pod %s(%s) is blacklisted", agent.PodName, agent.PodIp) return } diff --git a/cmd/traffic/cmd/manager/state/intercept.go b/cmd/traffic/cmd/manager/state/intercept.go index 53dbf5a2e9..7d399f1b51 100644 --- a/cmd/traffic/cmd/manager/state/intercept.go +++ b/cmd/traffic/cmd/manager/state/intercept.go @@ -768,10 +768,10 @@ func (s *state) waitForAgents(ctx context.Context, ac *agentconfig.Sidecar, fail } as := make([]*rpc.AgentInfo, 0, len(snapshot)) for _, a := range snapshot { - if mm.IsInactive(a.PodName) { - dlog.Debugf(ctx, "Agent %s.%s is blacklisted", a.PodName, a.Namespace) + if mm.IsInactive(a.PodIp) { + dlog.Debugf(ctx, "Agent %s(%s) is blacklisted", a.PodName, a.PodIp) } else { - dlog.Debugf(ctx, "Agent %s.%s is ready", a.Name, a.Namespace) + dlog.Debugf(ctx, "Agent %s(%s) is ready", a.Name, a.PodIp) as = append(as, a) break } diff --git a/cmd/traffic/cmd/manager/state/state.go b/cmd/traffic/cmd/manager/state/state.go index d0486b30d4..43e8a172ea 100644 --- a/cmd/traffic/cmd/manager/state/state.go +++ b/cmd/traffic/cmd/manager/state/state.go @@ -314,8 +314,8 @@ func (s *state) RemoveSession(ctx context.Context, sessionID string) { } func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *rpc.AgentInfo) { - dlog.Debugf(ctx, "Consolidating intercepts after removal of agent %s", agent.PodName) - mutator.GetMap(s.backgroundCtx).Inactivate(agent.PodName) + dlog.Debugf(ctx, "Consolidating intercepts after removal of agent %s(%s)", agent.PodName, agent.PodIp) + mutator.GetMap(s.backgroundCtx).Inactivate(agent.PodIp) s.intercepts.Range(func(interceptID string, intercept *rpc.InterceptInfo) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED || agent.PodIp != intercept.PodIp { // Not of interest. Continue iteration. @@ -331,7 +331,7 @@ func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *rp }) } else { // The agent is about to die, but apparently more agents are present. Let some other agent pick it up then. - dlog.Debugf(ctx, "Intercept %q lost its agent pod %s. Setting it disposition to WAITING", interceptID, agent.PodName) + dlog.Debugf(ctx, "Intercept %q lost its agent pod %s(%s). Setting it disposition to WAITING", interceptID, agent.PodName, agent.PodIp) s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { intercept.PodIp = "" intercept.PodName = "" @@ -447,10 +447,10 @@ func (s *state) CountTunnelEgress() uint64 { // Sessions: Agents //////////////////////////////////////////////////////////////////////////////// func (s *state) AddAgent(ctx context.Context, agent *rpc.AgentInfo, now time.Time) (string, error) { - if mutator.GetMap(ctx).IsInactive(agent.PodName) { + if mutator.GetMap(ctx).IsInactive(agent.PodIp) { return "", status.Error(codes.Aborted, "inactivated pod") } - sessionID := AgentSessionIDPrefix + agent.PodName + "." + agent.Namespace + sessionID := AgentSessionIDPrefix + agent.PodIp + "." + agent.Namespace if oldAgent, hasConflict := s.agents.LoadOrStore(sessionID, agent); hasConflict { return "", status.Error(codes.AlreadyExists, fmt.Sprintf("duplicate id %q, existing %+v, new %+v", sessionID, oldAgent, agent)) } @@ -481,7 +481,7 @@ func (s *state) AddAgent(ctx context.Context, agent *rpc.AgentInfo, now time.Tim func (s *state) GetAgent(sessionID string) *rpc.AgentInfo { if ret, ok := s.agents.Load(sessionID); ok { - if !mutator.GetMap(s.backgroundCtx).IsInactive(ret.PodName) { + if !mutator.GetMap(s.backgroundCtx).IsInactive(ret.PodIp) { return ret } } @@ -491,7 +491,7 @@ func (s *state) GetAgent(sessionID string) *rpc.AgentInfo { func (s *state) EachAgent(f func(string, *rpc.AgentInfo) bool) { m := mutator.GetMap(s.backgroundCtx) s.agents.Range(func(si string, ag *rpc.AgentInfo) bool { - if !m.IsInactive(ag.PodName) { + if !m.IsInactive(ag.PodIp) { return f(si, ag) } return true @@ -501,7 +501,7 @@ func (s *state) EachAgent(f func(string, *rpc.AgentInfo) bool) { func (s *state) LoadMatchingAgents(f func(string, *rpc.AgentInfo) bool) map[string]*rpc.AgentInfo { m := mutator.GetMap(s.backgroundCtx) return s.agents.LoadMatching(func(s string, ai *rpc.AgentInfo) bool { - return !m.IsInactive(ai.PodName) && f(s, ai) + return !m.IsInactive(ai.PodIp) && f(s, ai) }) } diff --git a/cmd/traffic/cmd/manager/state/workload_info_watcher.go b/cmd/traffic/cmd/manager/state/workload_info_watcher.go index e2a39b2dc2..78df3a3765 100644 --- a/cmd/traffic/cmd/manager/state/workload_info_watcher.go +++ b/cmd/traffic/cmd/manager/state/workload_info_watcher.go @@ -262,7 +262,7 @@ func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[ m := mutator.GetMap(ctx) for k, a := range oldAgentInfos { ai, ok := ais[k] - if !ok || m.IsInactive(ai.PodName) { + if !ok || m.IsInactive(ai.PodIp) { name := a.Name as := rpc.WorkloadInfo_NO_AGENT_UNSPECIFIED if w, ok := wf.workloadEvents[name]; ok && w.Type != rpc.WorkloadEvent_DELETED { @@ -292,7 +292,7 @@ func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[ } } for _, a := range ais { - if m.IsInactive(a.PodName) { + if m.IsInactive(a.PodIp) { continue } name := a.Name diff --git a/pkg/agentmap/discorvery.go b/pkg/agentmap/discorvery.go index 274c7f75c3..ef47ea2d5a 100644 --- a/pkg/agentmap/discorvery.go +++ b/pkg/agentmap/discorvery.go @@ -25,7 +25,7 @@ import ( var ReplicaSetNameRx = regexp.MustCompile(`\A(.+)-[a-f0-9]+\z`) func FindOwnerWorkload(ctx context.Context, obj k8sapi.Object, supportedWorkloadKinds []string) (k8sapi.Workload, error) { - dlog.Debugf(ctx, "FindOwnerWorkload(%s,%s,%s)", obj.GetName(), obj.GetNamespace(), obj.GetKind()) + dlog.Tracef(ctx, "FindOwnerWorkload(%s,%s,%s)", obj.GetName(), obj.GetNamespace(), obj.GetKind()) lbs := obj.GetLabels() if wlName, ok := lbs[agentconfig.WorkloadNameLabel]; ok { return GetWorkload(ctx, wlName, obj.GetNamespace(), lbs[agentconfig.WorkloadKindLabel]) @@ -62,7 +62,7 @@ func FindOwnerWorkload(ctx context.Context, obj k8sapi.Object, supportedWorkload } func GetWorkload(ctx context.Context, name, namespace, workloadKind string) (obj k8sapi.Workload, err error) { - dlog.Debugf(ctx, "GetWorkload(%s,%s,%s)", name, namespace, workloadKind) + dlog.Tracef(ctx, "GetWorkload(%s,%s,%s)", name, namespace, workloadKind) i := informer.GetFactory(ctx, namespace) if i == nil { dlog.Debugf(ctx, "fetching %s %s.%s using direct API call", workloadKind, name, namespace) @@ -200,7 +200,7 @@ func findServicesSelecting(ctx context.Context, namespace string, lbs labels.Lab } } else { // This shouldn't happen really. - dlog.Debugf(ctx, "Fetching services in %s using direct API call", namespace) + dlog.Tracef(ctx, "Fetching services in %s using direct API call", namespace) l, err := k8sapi.GetK8sInterface(ctx).CoreV1().Services(namespace).List(ctx, meta.ListOptions{}) if err != nil { return nil, err @@ -219,7 +219,7 @@ func findServicesSelecting(ctx context.Context, namespace string, lbs labels.Lab sort.Slice(ms, func(i, j int) bool { return ms[i].GetName() < ms[j].GetName() }) - dlog.Debugf(ctx, "Scanned %d services in namespace %s and found that %s selects labels %v", scanned, namespace, objectsStringer(ms), lbs) + dlog.Tracef(ctx, "Scanned %d services in namespace %s and found that %s selects labels %v", scanned, namespace, objectsStringer(ms), lbs) return ms, nil } From 0d4870f19a8aa5a1cc791e0f9d5bf426aac74ccc Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Tue, 28 Jan 2025 23:58:56 +0100 Subject: [PATCH 29/61] Reuse const for "traffic-manager". Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/manager/cluster/info.go | 2 -- cmd/traffic/cmd/manager/config/config.go | 3 ++- .../cmd/manager/mutator/agent_injector_test.go | 4 ++-- cmd/traffic/cmd/manager/service.go | 1 + cmd/traffic/cmd/manager/service_test.go | 2 +- integration_test/agent_injector_disabled_test.go | 5 +++-- integration_test/connected_test.go | 3 ++- integration_test/install_test.go | 11 ++++++----- integration_test/itest/cluster.go | 7 ++++--- integration_test/itest/namespace.go | 9 +++++---- integration_test/namespaces_test.go | 7 ++++--- integration_test/uninstall_test.go | 3 ++- pkg/client/cli/cmd/gather_logs.go | 5 +++-- pkg/client/cli/helm/install.go | 3 ++- pkg/client/userd/k8s/k8s_cluster.go | 3 ++- pkg/client/userd/k8s/outbound.go | 3 ++- pkg/client/userd/trafficmgr/gather_logs.go | 4 ++-- pkg/client/userd/trafficmgr/session.go | 5 +++-- 18 files changed, 46 insertions(+), 34 deletions(-) diff --git a/cmd/traffic/cmd/manager/cluster/info.go b/cmd/traffic/cmd/manager/cluster/info.go index b19ed3e84f..dfa04b9c76 100644 --- a/cmd/traffic/cmd/manager/cluster/info.go +++ b/cmd/traffic/cmd/manager/cluster/info.go @@ -29,8 +29,6 @@ import ( const ( supportedKubeAPIVersion = "1.17.0" - agentContainerName = "traffic-agent" - managerAppName = "traffic-manager" ) type Info interface { diff --git a/cmd/traffic/cmd/manager/config/config.go b/cmd/traffic/cmd/manager/config/config.go index 450d62a1c6..6a8b84828b 100644 --- a/cmd/traffic/cmd/manager/config/config.go +++ b/cmd/traffic/cmd/manager/config/config.go @@ -16,6 +16,7 @@ import ( "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/namespaces" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" "github.com/telepresenceio/telepresence/v2/pkg/labels" @@ -25,7 +26,7 @@ const ( clientConfigFileName = "client.yaml" agentEnvConfigFileName = "agent-env.yaml" namespaceSelectorConfigFileName = "namespace-selector.yaml" - cfgConfigMapName = "traffic-manager" + cfgConfigMapName = agentmap.ManagerAppName ) type Watcher interface { diff --git a/cmd/traffic/cmd/manager/mutator/agent_injector_test.go b/cmd/traffic/cmd/manager/mutator/agent_injector_test.go index 490c93f5a2..ba25e036ca 100644 --- a/cmd/traffic/cmd/manager/mutator/agent_injector_test.go +++ b/cmd/traffic/cmd/manager/mutator/agent_injector_test.go @@ -56,7 +56,7 @@ const mgrNs = "default" func TestTrafficAgentConfigGenerator(t *testing.T) { managerConfig := core.ConfigMap{ ObjectMeta: meta.ObjectMeta{ - Name: "traffic-manager", + Name: agentmap.ManagerAppName, Namespace: mgrNs, }, Data: map[string]string{"namespace-selector.yaml": ` @@ -912,7 +912,7 @@ func TestTrafficAgentInjector(t *testing.T) { one := int32(1) managerConfig := core.ConfigMap{ ObjectMeta: meta.ObjectMeta{ - Name: "traffic-manager", + Name: agentmap.ManagerAppName, Namespace: mgrNs, }, Data: map[string]string{"namespace-selector.yaml": ` diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index cb7b7f2304..de1eb8bc30 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -26,6 +26,7 @@ import ( "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/managerutil" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/mutator" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/state" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/dnsproxy" "github.com/telepresenceio/telepresence/v2/pkg/tunnel" "github.com/telepresenceio/telepresence/v2/pkg/version" diff --git a/cmd/traffic/cmd/manager/service_test.go b/cmd/traffic/cmd/manager/service_test.go index e015df853b..132e32eb29 100644 --- a/cmd/traffic/cmd/manager/service_test.go +++ b/cmd/traffic/cmd/manager/service_test.go @@ -233,7 +233,7 @@ func getTestClientConn(ctx context.Context, t *testing.T) *grpc.ClientConn { _, err = fakeClient.CoreV1().ConfigMaps(mgrNs).Create(ctx, &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: "traffic-manager", + Name: agentmap.ManagerAppName, Namespace: mgrNs, }, Data: map[string]string{"namespace-selector.yaml": ` diff --git a/integration_test/agent_injector_disabled_test.go b/integration_test/agent_injector_disabled_test.go index 941a31e678..b25b645ec8 100644 --- a/integration_test/agent_injector_disabled_test.go +++ b/integration_test/agent_injector_disabled_test.go @@ -4,6 +4,7 @@ import ( "os" "github.com/telepresenceio/telepresence/v2/integration_test/itest" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" ) type agentInjectorDisabledSuite struct { @@ -56,10 +57,10 @@ func (s *agentInjectorDisabledSuite) Test_VersionWithAgentInjectorDisabled() { ctx := s.Context() rq := s.Require() restartCount := func() int { - pods := itest.RunningPods(ctx, "traffic-manager", s.ManagerNamespace()) + pods := itest.RunningPods(ctx, agentmap.ManagerAppName, s.ManagerNamespace()) if len(pods) == 1 { for _, cs := range pods[0].Status.ContainerStatuses { - if cs.Name == "traffic-manager" { + if cs.Name == agentmap.ManagerAppName { return int(cs.RestartCount) } } diff --git a/integration_test/connected_test.go b/integration_test/connected_test.go index 09b014afca..5f8ce30bdb 100644 --- a/integration_test/connected_test.go +++ b/integration_test/connected_test.go @@ -5,6 +5,7 @@ import ( "regexp" "github.com/telepresenceio/telepresence/v2/integration_test/itest" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" ) type connectedSuite struct { @@ -24,7 +25,7 @@ func init() { func (s *connectedSuite) Test_ListExcludesTM() { stdout := itest.TelepresenceOk(s.Context(), "list", "-n", s.ManagerNamespace()) - s.NotContains(stdout, "traffic-manager") + s.NotContains(stdout, agentmap.ManagerAppName) } func (s *connectedSuite) Test_ReportsAllVersions() { diff --git a/integration_test/install_test.go b/integration_test/install_test.go index 298f18f58a..01a6168fd4 100644 --- a/integration_test/install_test.go +++ b/integration_test/install_test.go @@ -18,6 +18,7 @@ import ( "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/integration_test/itest" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/client/cli/helm" "github.com/telepresenceio/telepresence/v2/pkg/client/userd/k8s" @@ -26,7 +27,7 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/version" ) -const ManagerAppName = "traffic-manager" +const ManagerAppName = agentmap.ManagerAppName type installSuite struct { itest.Suite @@ -66,7 +67,7 @@ func (is *installSuite) Test_UpgradeRetainsValues() { rq.NoError(err) getValues := func() (map[string]any, error) { - return action.NewGetValues(helmConfig).Run("traffic-manager") + return action.NewGetValues(helmConfig).Run(agentmap.ManagerAppName) } containsKey := func(m map[string]any, key string) bool { _, ok := m[key] @@ -131,7 +132,7 @@ func (is *installSuite) Test_HelmTemplateInstall() { "managerRbac.create": true, }, false) require.NoError(err) - values = append([]string{"template", "traffic-manager", chart, "-n", is.ManagerNamespace()}, values...) + values = append([]string{"template", agentmap.ManagerAppName, chart, "-n", is.ManagerNamespace()}, values...) manifest, err := itest.Output(ctx, "helm", values...) require.NoError(err) out := dlog.StdLogger(ctx, dlog.LogLevelInfo).Writer() @@ -141,8 +142,8 @@ func (is *installSuite) Test_HelmTemplateInstall() { // Sometimes the traffic-agents configmap gets wiped, causing the delete command to fail, hence we don't require.NoError _ = itest.Kubectl(dos.WithStdin(logCtx, strings.NewReader(manifest)), "", "delete", "-f", "-") }() - require.NoError(itest.RolloutStatusWait(ctx, is.ManagerNamespace(), "deploy/traffic-manager")) - is.CapturePodLogs(ctx, "traffic-manager", "", is.ManagerNamespace()) + require.NoError(itest.RolloutStatusWait(ctx, is.ManagerNamespace(), "deploy/"+agentmap.ManagerAppName)) + is.CapturePodLogs(ctx, agentmap.ManagerAppName, "", is.ManagerNamespace()) stdout := is.TelepresenceConnect(ctx) is.Contains(stdout, "Connected to context") itest.TelepresenceQuitOk(ctx) diff --git a/integration_test/itest/cluster.go b/integration_test/itest/cluster.go index ca21c7a66c..ed2ef738e1 100644 --- a/integration_test/itest/cluster.go +++ b/integration_test/itest/cluster.go @@ -39,6 +39,7 @@ import ( "github.com/datawire/dtest" telcharts "github.com/telepresenceio/telepresence/v2/charts" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/client/socket" "github.com/telepresenceio/telepresence/v2/pkg/client/userd/k8s" @@ -805,10 +806,10 @@ func (s *cluster) TelepresenceHelmInstall(ctx context.Context, upgrade bool, set if _, _, err = Telepresence(WithUser(ctx, "default"), args...); err != nil { return "", err } - if err = RolloutStatusWait(ctx, nss.Namespace, "deploy/traffic-manager"); err != nil { + if err = RolloutStatusWait(ctx, nss.Namespace, "deploy/"+agentmap.ManagerAppName); err != nil { return "", err } - logFileName := s.self.CapturePodLogs(ctx, "traffic-manager", "", nss.Namespace) + logFileName := s.self.CapturePodLogs(ctx, agentmap.ManagerAppName, "", nss.Namespace) return logFileName, nil } @@ -832,7 +833,7 @@ func (s *cluster) UninstallTrafficManager(ctx context.Context, managerNamespace TelepresenceOk(ctx, append([]string{"helm", "uninstall", "--manager-namespace", managerNamespace}, args...)...) // Helm uninstall does deletions asynchronously, so let's wait until the deployment is gone - assert.Eventually(t, func() bool { return len(RunningPodNames(ctx, "traffic-manager", managerNamespace)) == 0 }, + assert.Eventually(t, func() bool { return len(RunningPodNames(ctx, agentmap.ManagerAppName, managerNamespace)) == 0 }, 60*time.Second, 4*time.Second, "traffic-manager deployment was not removed") TelepresenceQuitOk(ctx) } diff --git a/integration_test/itest/namespace.go b/integration_test/itest/namespace.go index fcc10f9cf4..d9b4bd4f50 100644 --- a/integration_test/itest/namespace.go +++ b/integration_test/itest/namespace.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/datawire/dlib/dlog" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/dos" "github.com/telepresenceio/telepresence/v2/pkg/labels" ) @@ -132,14 +133,14 @@ func (s *nsPair) tearDown(ctx context.Context) { func (s *nsPair) RollbackTM(ctx context.Context) { ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() - err := Command(ctx, "helm", "rollback", "--no-hooks", "--wait", "--namespace", s.ManagerNamespace(), "traffic-manager").Run() + err := Command(ctx, "helm", "rollback", "--no-hooks", "--wait", "--namespace", s.ManagerNamespace(), agentmap.ManagerAppName).Run() t := getT(ctx) require.NoError(t, err) - require.NoError(t, RolloutStatusWait(ctx, s.Namespace, "deploy/traffic-manager")) + require.NoError(t, RolloutStatusWait(ctx, s.Namespace, "deploy/"+agentmap.ManagerAppName)) assert.Eventually(t, func() bool { - return len(RunningPodNames(ctx, "traffic-manager", s.Namespace)) == 1 + return len(RunningPodNames(ctx, agentmap.ManagerAppName, s.Namespace)) == 1 }, 30*time.Second, 5*time.Second) - s.CapturePodLogs(ctx, "traffic-manager", "", s.Namespace) + s.CapturePodLogs(ctx, agentmap.ManagerAppName, "", s.Namespace) } func (s *nsPair) AppNamespace() string { diff --git a/integration_test/namespaces_test.go b/integration_test/namespaces_test.go index adc8a53031..f9ebfd5993 100644 --- a/integration_test/namespaces_test.go +++ b/integration_test/namespaces_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/telepresenceio/telepresence/v2/integration_test/itest" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/dos" "github.com/telepresenceio/telepresence/v2/pkg/labels" ) @@ -191,10 +192,10 @@ func (s *nsSuite) Test_NamespacesDynamic() { }) restartCount := func() int { - pods := itest.RunningPods(ctx, "traffic-manager", s.managerNamespace()) + pods := itest.RunningPods(ctx, agentmap.ManagerAppName, s.managerNamespace()) if len(pods) == 1 { for _, cs := range pods[0].Status.ContainerStatuses { - if cs.Name == "traffic-manager" { + if cs.Name == agentmap.ManagerAppName { return int(cs.RestartCount) } } @@ -276,7 +277,7 @@ func (s *nsSuite) Test_NamespacesStatic() { getPodName := func() (podName string) { rq.Eventually(func() bool { - pods := itest.RunningPods(ctx, "traffic-manager", s.managerNamespace()) + pods := itest.RunningPods(ctx, agentmap.ManagerAppName, s.managerNamespace()) if len(pods) == 1 { podName = pods[0].Name return true diff --git a/integration_test/uninstall_test.go b/integration_test/uninstall_test.go index 4ae50335a8..96b2d1d902 100644 --- a/integration_test/uninstall_test.go +++ b/integration_test/uninstall_test.go @@ -7,6 +7,7 @@ import ( "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/integration_test/itest" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" ) func (s *notConnectedSuite) Test_Uninstall() { @@ -17,7 +18,7 @@ func (s *notConnectedSuite) Test_Uninstall() { names := func() (string, error) { return itest.KubectlOut(ctx, s.ManagerNamespace(), - "get", "svc,deploy", "traffic-manager", + "get", "svc,deploy", agentmap.ManagerAppName, "--ignore-not-found", "-o", "jsonpath={.items[*].metadata.name}") } diff --git a/pkg/client/cli/cmd/gather_logs.go b/pkg/client/cli/cmd/gather_logs.go index b0a5bbae42..efdbbd3422 100644 --- a/pkg/client/cli/cmd/gather_logs.go +++ b/pkg/client/cli/cmd/gather_logs.go @@ -17,6 +17,7 @@ import ( "google.golang.org/grpc" "github.com/telepresenceio/telepresence/rpc/v2/connector" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/client/cli/ann" "github.com/telepresenceio/telepresence/v2/pkg/client/cli/connect" @@ -436,8 +437,8 @@ func (a *anonymizer) getPodName(podName string) string { // we want to special case the traffic-manager so we can easily distinguish // between that and the traffic-agents - if strings.Contains(name, "traffic-manager") { - anonPodName = fmt.Sprintf("traffic-manager.%s", anonNamespace) + if strings.Contains(name, agentmap.ManagerAppName) { + anonPodName = fmt.Sprintf("%s.%s", agentmap.ManagerAppName, anonNamespace) } else { anonPodName = fmt.Sprintf("pod-%d.%s", len(a.podNames)+1, anonNamespace) } diff --git a/pkg/client/cli/helm/install.go b/pkg/client/cli/helm/install.go index a9a06dfda5..e04da12a90 100644 --- a/pkg/client/cli/helm/install.go +++ b/pkg/client/cli/helm/install.go @@ -20,6 +20,7 @@ import ( "github.com/datawire/dlib/dlog" "github.com/datawire/dlib/dtime" "github.com/telepresenceio/telepresence/rpc/v2/connector" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/client/userd/k8s" "github.com/telepresenceio/telepresence/v2/pkg/dos" @@ -30,7 +31,7 @@ import ( const ( helmDriver = "secrets" - trafficManagerReleaseName = "traffic-manager" + trafficManagerReleaseName = agentmap.ManagerAppName crdReleaseName = "telepresence-crds" ) diff --git a/pkg/client/userd/k8s/k8s_cluster.go b/pkg/client/userd/k8s/k8s_cluster.go index a2e6bfbb4e..10143d45a8 100644 --- a/pkg/client/userd/k8s/k8s_cluster.go +++ b/pkg/client/userd/k8s/k8s_cluster.go @@ -17,6 +17,7 @@ import ( "github.com/datawire/dlib/dlog" "github.com/datawire/dlib/dtime" rpc "github.com/telepresenceio/telepresence/rpc/v2/connector" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/client/userd" "github.com/telepresenceio/telepresence/v2/pkg/errcat" @@ -242,7 +243,7 @@ func (kc *Cluster) determineTrafficManagerNamespace(c context.Context) (string, // Search for the traffic-manager in mapped namespaces nss := kc.GetCurrentNamespaces(true) for _, ns := range nss { - if _, err := k8sapi.GetService(c, "traffic-manager", ns); err == nil { + if _, err := k8sapi.GetService(c, agentmap.ManagerAppName, ns); err == nil { return ns, nil } } diff --git a/pkg/client/userd/k8s/outbound.go b/pkg/client/userd/k8s/outbound.go index a6d3af4a14..e6251e3ff0 100644 --- a/pkg/client/userd/k8s/outbound.go +++ b/pkg/client/userd/k8s/outbound.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "github.com/datawire/dlib/dlog" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/client/userd" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" ) @@ -85,7 +86,7 @@ func canGetDefaultTrafficManagerService(ctx context.Context) bool { ok, err := k8sapi.CanI(ctx, &auth.ResourceAttributes{ Verb: "get", Resource: "services", - Name: "traffic-manager", + Name: agentmap.ManagerAppName, Namespace: defaultManagerNamespace, }) return err == nil && ok diff --git a/pkg/client/userd/trafficmgr/gather_logs.go b/pkg/client/userd/trafficmgr/gather_logs.go index 9085b7e5de..3775896104 100644 --- a/pkg/client/userd/trafficmgr/gather_logs.go +++ b/pkg/client/userd/trafficmgr/gather_logs.go @@ -148,7 +148,7 @@ func (s *session) GatherLogs(ctx context.Context, request *connector.LogsRequest ns := k8s.GetManagerNamespace(ctx) podsAPI := coreAPI.Pods(ns) selector := labels.SelectorFromSet(labels.Set{ - "app": "traffic-manager", + "app": agentmap.ManagerAppName, "telepresence": "manager", }) podList, err := podsAPI.List(ctx, meta.ListOptions{LabelSelector: selector.String()}) @@ -161,7 +161,7 @@ func (s *session) GatherLogs(ctx context.Context, request *connector.LogsRequest pod := &podList.Items[0] podAndNs := fmt.Sprintf("%s.%s", pod.Name, ns) dlog.Debugf(ctx, "gathering logs for %s, yaml = %t", podAndNs, request.GetPodYaml) - getPodLog(ctx, exportDir, &result, podsAPI, pod, "traffic-manager", request.GetPodYaml, false) + getPodLog(ctx, exportDir, &result, podsAPI, pod, agentmap.ManagerAppName, request.GetPodYaml, false) case len(podList.Items) > 1: err = fmt.Errorf("multiple traffic managers found in namespace %s using selector %s", ns, selector.String()) dlog.Error(ctx, err) diff --git a/pkg/client/userd/trafficmgr/session.go b/pkg/client/userd/trafficmgr/session.go index 78a07073f6..73b3be7b85 100644 --- a/pkg/client/userd/trafficmgr/session.go +++ b/pkg/client/userd/trafficmgr/session.go @@ -40,6 +40,7 @@ import ( rootdRpc "github.com/telepresenceio/telepresence/rpc/v2/daemon" "github.com/telepresenceio/telepresence/rpc/v2/manager" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" + "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/authenticator/patcher" "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/client/cli/daemon" @@ -481,8 +482,8 @@ func (s *session) Remain(ctx context.Context) error { func CheckTrafficManagerService(ctx context.Context, namespace string) error { dlog.Debug(ctx, "checking that traffic-manager exists") coreV1 := k8sapi.GetK8sInterface(ctx).CoreV1() - if _, err := coreV1.Services(namespace).Get(ctx, "traffic-manager", meta.GetOptions{}); err != nil { - msg := fmt.Sprintf("unable to get service traffic-manager in %s: %v", namespace, err) + if _, err := coreV1.Services(namespace).Get(ctx, agentmap.ManagerAppName, meta.GetOptions{}); err != nil { + msg := fmt.Sprintf("unable to get service %s in %s: %v", agentmap.ManagerAppName, namespace, err) se := &k8serrors.StatusError{} if errors.As(err, &se) { if se.Status().Code == http.StatusNotFound { From 3f648e53143f828cf307d55066a5e89626f1a78b Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Wed, 29 Jan 2025 02:01:43 +0100 Subject: [PATCH 30/61] DNS optimizations on the traffic-manager side. Let traffic-manager short-circuit DNS requests to itself. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/agent/client.go | 2 +- cmd/traffic/cmd/manager/cluster/info.go | 5 + cmd/traffic/cmd/manager/service.go | 127 ++++++++++++++++++++---- pkg/dnsproxy/lookup.go | 20 ++-- pkg/dnsproxy/lookup_test.go | 2 +- 5 files changed, 126 insertions(+), 30 deletions(-) diff --git a/cmd/traffic/cmd/agent/client.go b/cmd/traffic/cmd/agent/client.go index f343172d66..6d27829f58 100644 --- a/cmd/traffic/cmd/agent/client.go +++ b/cmd/traffic/cmd/agent/client.go @@ -234,7 +234,7 @@ func lookupDNSWaitLoop(ctx context.Context, manager rpc.ManagerClient, session * func lookupDNSAndRespond(ctx context.Context, manager rpc.ManagerClient, session *rpc.SessionInfo, lr *rpc.DNSRequest) { qType := uint16(lr.Type) tqn := dns2.TypeToString[qType] - rrs, rCode, err := dnsproxy.Lookup(ctx, qType, lr.Name) + rrs, rCode, err := dnsproxy.Lookup(ctx, qType, lr.Name, "") if err != nil { dlog.Errorf(ctx, "LookupDNS %s %s: %v", lr.Name, tqn, err) return diff --git a/cmd/traffic/cmd/manager/cluster/info.go b/cmd/traffic/cmd/manager/cluster/info.go index dfa04b9c76..71a050045a 100644 --- a/cmd/traffic/cmd/manager/cluster/info.go +++ b/cmd/traffic/cmd/manager/cluster/info.go @@ -38,6 +38,8 @@ type Info interface { // ID of the installed ns ID() string + ServiceIP() net.IP + // SetAdditionalAlsoProxy assigns a slice that will be added to the Routing.AlsoProxySubnets slice // when notifications are sent. SetAdditionalAlsoProxy(ctx context.Context, subnets []*rpc.IPNet) @@ -403,6 +405,9 @@ func (oi *info) ID() string { return oi.installID } +func (oi *info) ServiceIP() net.IP { + return oi.InjectorSvcIp +} func (oi *info) ClusterDomain() string { return oi.Dns.ClusterDomain } diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index de1eb8bc30..699eb3139d 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "maps" + "net" "net/netip" "slices" "sort" + "strings" "time" "github.com/blang/semver/v4" @@ -62,6 +64,9 @@ type service struct { configWatcher config.Watcher activeHttpRequests int32 activeGrpcRequests int32 + serviceNameNs string + serviceNameFQN string + dotClusterDomain string // Possibly extended version of the service. Use when calling interface methods. self Service @@ -104,6 +109,12 @@ func NewService(ctx context.Context, configWatcher config.Watcher) (Service, *dg // These are context dependent so build them once the pool is up ret.clusterInfo = cluster.NewInfo(ctx) ret.state = state.NewStateFunc(ctx) + + ns := managerutil.GetEnv(ctx).ManagerNamespace + ret.dotClusterDomain = "." + ret.clusterInfo.ClusterDomain() + ret.serviceNameNs = fmt.Sprintf("%s.%s.", agentmap.ManagerAppName, ns) + ret.serviceNameFQN = fmt.Sprintf("%s.%s.svc%s", agentmap.ManagerAppName, ns, ret.dotClusterDomain) + ret.self = ret g := dgroup.NewGroup(ctx, dgroup.GroupConfig{ EnableSignalHandling: true, @@ -799,24 +810,99 @@ func hasDomainSuffix(name, suffix string) bool { return name[sfp-1] == '.' && name[sfp:] == suffix } -func (s *service) LookupDNS(ctx context.Context, request *rpc.DNSRequest) (*rpc.DNSResponse, error) { +func (s *service) resolveSelfDNS(svcIP net.IP, request *rpc.DNSRequest) (rrs dnsproxy.RRs) { + qType := uint16(request.Type) + switch qType { + case dns2.TypeA: + rrs = dnsproxy.RRs{&dns2.A{ + Hdr: dns2.RR_Header{ + Name: request.Name, + Rrtype: qType, + Class: dns2.ClassINET, + }, + A: svcIP.To4(), + }} + case dns2.TypeAAAA: + var ip net.IP + if svcIP.To4() == nil { + ip = svcIP.To16() + } + rrs = dnsproxy.RRs{&dns2.AAAA{ + Hdr: dns2.RR_Header{ + Name: request.Name, + Rrtype: qType, + Class: dns2.ClassINET, + }, + AAAA: ip, + }} + case dns2.TypeCNAME: + rrs = dnsproxy.RRs{&dns2.CNAME{ + Hdr: dns2.RR_Header{ + Name: request.Name, + Rrtype: qType, + Class: dns2.ClassINET, + }, + Target: s.serviceNameFQN, + }} + } + return rrs +} + +func (s *service) LookupDNS(ctx context.Context, request *rpc.DNSRequest) (response *rpc.DNSResponse, err error) { ctx = managerutil.WithSessionInfo(ctx, request.GetSession()) qType := uint16(request.Type) qtn := dns2.TypeToString[qType] - dlog.Debugf(ctx, "LookupDNS %s %s", request.Name, qtn) + var rrs dnsproxy.RRs + if dlog.MaxLogLevel(ctx) >= dlog.LogLevelDebug { + defer func() { + var result string + switch { + case err != nil: + result = err.Error() + case len(rrs) == 0: + result = dns2.RcodeToString[int(response.RCode)] + default: + result = rrs.String() + } + dlog.Debugf(ctx, "LookupDNS: %s %s -> %s", request.Name, qtn, result) + }() + } - rrs, rCode, err := s.state.AgentsLookupDNS(ctx, request.GetSession().GetSessionId(), request) - if err != nil { - dlog.Errorf(ctx, "AgentsLookupDNS %s %s: %v", request.Name, qtn, err) - } else if rCode != state.RcodeNoAgents { - if len(rrs) == 0 { - dlog.Debugf(ctx, "LookupDNS on agents: %s %s -> %s", request.Name, qtn, dns2.RcodeToString[rCode]) - } else { - dlog.Debugf(ctx, "LookupDNS on agents: %s %s -> %s", request.Name, qtn, rrs) + if svcIP := s.clusterInfo.ServiceIP(); svcIP != nil && (request.Name == s.serviceNameNs || request.Name == s.serviceNameFQN) { + rrs = s.resolveSelfDNS(svcIP, request) + if len(rrs) > 0 { + return dnsproxy.ToRPC(rrs, dns2.RcodeSuccess) } } + + tmNamespace := managerutil.GetEnv(ctx).ManagerNamespace + noSearchDomain := s.dotClusterDomain + var rCode int + switch { + case request.Name == "tel2-recursion-check.kube-system.": + rCode = state.RcodeNoAgents + noSearchDomain = ".kube-system." + case hasDomainSuffix(request.Name, tmNamespace): + // It's enough to propagate this one to the traffic-manager + noSearchDomain = tmNamespace + "." + rCode = state.RcodeNoAgents + case strings.HasSuffix(request.Name, s.dotClusterDomain): + // It's enough to propagate this one to the traffic-manager + rCode = state.RcodeNoAgents + default: + rrs, rCode, err = s.state.AgentsLookupDNS(ctx, request.GetSession().GetSessionId(), request) + if err != nil { + dlog.Errorf(ctx, "AgentsLookupDNS %s %s: %v", request.Name, qtn, err) + } else if rCode != state.RcodeNoAgents { + if len(rrs) == 0 { + dlog.Tracef(ctx, "LookupDNS on agents: %s %s -> %s", request.Name, qtn, dns2.RcodeToString[rCode]) + } else { + dlog.Tracef(ctx, "LookupDNS on agents: %s %s -> %s", request.Name, qtn, rrs) + } + } + } + if rCode == state.RcodeNoAgents { - tmNamespace := managerutil.GetEnv(ctx).ManagerNamespace client := s.state.GetClient(request.GetSession().GetSessionId()) name := request.Name restoreName := false @@ -828,35 +914,36 @@ func (s *service) LookupDNS(ctx context.Context, request *rpc.DNSRequest) (*rpc. } } if nDots == 1 && client.Namespace != tmNamespace { - name += client.Namespace + "." + noSearchDomain = client.Namespace + "." + name += noSearchDomain restoreName = true } } - dlog.Debugf(ctx, "LookupDNS on traffic-manager: %s", name) - rrs, rCode, err = dnsproxy.Lookup(ctx, qType, name) + dlog.Tracef(ctx, "LookupDNS on traffic-manager: %s", name) + rrs, rCode, err = dnsproxy.Lookup(ctx, qType, name, noSearchDomain) if err != nil { // Could still be x.y., but let's avoid x.. and x.. - if client != nil && nDots > 1 && client.Namespace != tmNamespace && !hasDomainSuffix(name, s.ClusterInfo().ClusterDomain()) && !hasDomainSuffix(name, client.Namespace) { + if client != nil && nDots > 1 && client.Namespace != tmNamespace && !strings.HasSuffix(name, s.dotClusterDomain) && !hasDomainSuffix(name, client.Namespace) { name += client.Namespace + "." restoreName = true dlog.Debugf(ctx, "LookupDNS on traffic-manager: %s", name) - rrs, rCode, err = dnsproxy.Lookup(ctx, qType, name) + rrs, rCode, err = dnsproxy.Lookup(ctx, qType, name, noSearchDomain) } if err != nil { - dlog.Debugf(ctx, "LookupDNS on traffic-manager: %s %s -> %s %s", request.Name, qtn, dns2.RcodeToString[rCode], err) + dlog.Tracef(ctx, "LookupDNS on traffic-manager: %s %s -> %s %s", request.Name, qtn, dns2.RcodeToString[rCode], err) return nil, err } } if len(rrs) == 0 { - dlog.Debugf(ctx, "LookupDNS on traffic-manager: %s %s -> %s", request.Name, qtn, dns2.RcodeToString[rCode]) + dlog.Tracef(ctx, "LookupDNS on traffic-manager: %s %s -> %s", request.Name, qtn, dns2.RcodeToString[rCode]) } else { if restoreName { - dlog.Debugf(ctx, "LookupDNS on traffic-manager: restore %s to %s", name, request.Name) + dlog.Tracef(ctx, "LookupDNS on traffic-manager: restore %s to %s", name, request.Name) for _, rr := range rrs { rr.Header().Name = request.Name } } - dlog.Debugf(ctx, "LookupDNS on traffic-manager: %s %s -> %s", request.Name, qtn, rrs) + dlog.Tracef(ctx, "LookupDNS on traffic-manager: %s %s -> %s", request.Name, qtn, rrs) } } return dnsproxy.ToRPC(rrs, rCode) diff --git a/pkg/dnsproxy/lookup.go b/pkg/dnsproxy/lookup.go index 9a5a221792..f76dc02acc 100644 --- a/pkg/dnsproxy/lookup.go +++ b/pkg/dnsproxy/lookup.go @@ -138,10 +138,14 @@ func NewHeader(qName string, qType uint16) dns.RR_Header { } // useLookupName takes care of an undocumented "feature" in some lookup functions. -// If the name ends with a dot, then no search path will be applied. If however, -// the name doesn't end with a dot, the search path is always applied and the name +// If the name ends with a dot, then no search path will be applied. If, however, +// the name doesn't end with a dot, the search path is always applied, and the name // is never used verbatim. -func useLookupName(qName string) (string, bool) { +// A name ending with noSearchDomain will always be considered final. +func useLookupName(qName, noSearchDomain string) (string, bool) { + if noSearchDomain != "" && strings.HasSuffix(qName, noSearchDomain) { + return qName, true + } dots := 0 name := qName[:len(qName)-1] for _, c := range qName { @@ -162,8 +166,8 @@ func useLookupName(qName string) (string, bool) { } } -func lookupIP(ctx context.Context, network, qName string, r *net.Resolver) ([]net.IP, error) { - name, final := useLookupName(qName) +func lookupIP(ctx context.Context, network, qName, noSearchDomain string, r *net.Resolver) ([]net.IP, error) { + name, final := useLookupName(qName, noSearchDomain) ips, err := r.LookupIP(ctx, network, name) if err != nil && !final { dlog.Errorf(ctx, "LookupIP failed, trying LookupIP %q", qName) @@ -194,12 +198,12 @@ func makeError(err error) (RRs, int, error) { return nil, dns.RcodeServerFailure, status.Error(codes.Internal, err.Error()) } -func Lookup(ctx context.Context, qType uint16, qName string) (RRs, int, error) { +func Lookup(ctx context.Context, qType uint16, qName, noSearchDomain string) (RRs, int, error) { var answer RRs r := &net.Resolver{StrictErrors: true} switch qType { case dns.TypeA, dns.TypeAAAA: - ips, err := lookupIP(ctx, "ip", qName, r) + ips, err := lookupIP(ctx, "ip", qName, noSearchDomain, r) if err != nil { return makeError(err) } @@ -235,7 +239,7 @@ func Lookup(ctx context.Context, qType uint16, qName string) (RRs, int, error) { } } case dns.TypeCNAME: - name, final := useLookupName(qName) + name, final := useLookupName(qName, noSearchDomain) target, err := r.LookupCNAME(ctx, name) if err != nil && !final { target, err = r.LookupCNAME(ctx, qName) diff --git a/pkg/dnsproxy/lookup_test.go b/pkg/dnsproxy/lookup_test.go index 63c45d1135..972de033c1 100644 --- a/pkg/dnsproxy/lookup_test.go +++ b/pkg/dnsproxy/lookup_test.go @@ -56,7 +56,7 @@ func TestLookup(t *testing.T) { t.Skip("SRV sporadically fails to parse reply on darwin") } ctx := dlog.NewTestContext(t, false) - got, _, err := Lookup(ctx, tt.qType, tt.qName) + got, _, err := Lookup(ctx, tt.qType, tt.qName, "") require.NoError(t, err) require.Greater(t, len(got), 0) }) From 40859c1b481835d7dd69fec2ed7ee0fb469b9899 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Thu, 30 Jan 2025 04:26:46 +0100 Subject: [PATCH 31/61] Fix mount sync problem when intercept ended. Signed-off-by: Thomas Hallgren --- integration_test/itest/cluster.go | 2 + integration_test/itest/template.go | 8 ++ integration_test/large_files_test.go | 18 +---- integration_test/mounts_test.go | 8 +- .../testdata/k8s/hello-pv-volume.goyaml | 74 ++++++++++++++++++ .../testdata/k8s/hello-pv-volume.yaml | 75 ------------------- pkg/client/userd/trafficmgr/ingest.go | 4 + pkg/client/userd/trafficmgr/intercept.go | 33 ++++++-- pkg/client/userd/trafficmgr/podaccess.go | 4 +- 9 files changed, 127 insertions(+), 99 deletions(-) create mode 100644 integration_test/testdata/k8s/hello-pv-volume.goyaml delete mode 100644 integration_test/testdata/k8s/hello-pv-volume.yaml diff --git a/integration_test/itest/cluster.go b/integration_test/itest/cluster.go index ed2ef738e1..603cad29d6 100644 --- a/integration_test/itest/cluster.go +++ b/integration_test/itest/cluster.go @@ -233,6 +233,8 @@ func (s *cluster) tearDown(ctx context.Context) { ctx = WithWorkingDir(ctx, GetOSSRoot(ctx)) _ = Run(ctx, "kubectl", "delete", "-f", filepath.Join("testdata", "k8s", "client_rbac.yaml")) _ = Run(ctx, "kubectl", "delete", "--wait=false", "ns", "-l", "purpose=tp-cli-testing") + _ = Run(ctx, "kubectl", "delete", "--wait=false", "pv", "-l", "purpose=tp-cli-testing") + _ = Run(ctx, "kubectl", "delete", "--wait=false", "storageclass", "-l", "purpose=tp-cli-testing") } } diff --git a/integration_test/itest/template.go b/integration_test/itest/template.go index 047e13123d..65c7e3af0d 100644 --- a/integration_test/itest/template.go +++ b/integration_test/itest/template.go @@ -39,6 +39,14 @@ type Generic struct { ServiceAccount string } +type PersistentVolume struct { + // Deployment and service name + Name string + + // MountDirectory in the pod + MountDirectory string +} + func OpenTemplate(ctx context.Context, name string, data any) (io.Reader, error) { b, err := ReadTemplate(ctx, name, data) if err != nil { diff --git a/integration_test/large_files_test.go b/integration_test/large_files_test.go index 9c120a4a88..64573bcd5e 100644 --- a/integration_test/large_files_test.go +++ b/integration_test/large_files_test.go @@ -32,12 +32,6 @@ type largeFilesSuite struct { largeFiles []string } -type qname struct { - Name string - Namespace string - VolumeSize string -} - func (s *largeFilesSuite) SuiteName() string { return "LargeFiles" } @@ -84,10 +78,9 @@ func (s *largeFilesSuite) SetupSuite() { go func(i int) { defer wg.Done() svc := fmt.Sprintf("%s-%d", s.Name(), i) - rdr, err := itest.OpenTemplate(ctx, filepath.Join(k8s, "hello-pv-volume.yaml"), &qname{ - Name: svc, - Namespace: s.AppNamespace(), - VolumeSize: "350Mi", // must cover fileCountPerSvc * fileSize + rdr, err := itest.OpenTemplate(ctx, filepath.Join(k8s, "hello-pv-volume.goyaml"), &itest.PersistentVolume{ + Name: svc, + MountDirectory: "/home/scratch", }) require.NoError(err) require.NoError(s.Kubectl(dos.WithStdin(ctx, rdr), "apply", "-f", "-")) @@ -134,15 +127,12 @@ func (s *largeFilesSuite) leaveIntercepts(ctx context.Context) { func (s *largeFilesSuite) TearDownSuite() { ctx := s.Context() - k8s := filepath.Join("testdata", "k8s") wg := sync.WaitGroup{} wg.Add(s.ServiceCount()) for i := 0; i < s.ServiceCount(); i++ { go func(i int) { defer wg.Done() - rdr, err := itest.OpenTemplate(ctx, filepath.Join(k8s, "hello-pv-volume.yaml"), &qname{Name: fmt.Sprintf("%s-%d", s.Name(), i), Namespace: s.AppNamespace()}) - s.NoError(err) - s.NoError(s.Kubectl(dos.WithStdin(ctx, rdr), "delete", "-f", "-")) + s.DeleteSvcAndWorkload(ctx, "deploy", fmt.Sprintf("%s-%d", s.Name(), i)) }(i) } itest.TelepresenceQuitOk(ctx) diff --git a/integration_test/mounts_test.go b/integration_test/mounts_test.go index 5fba1612d5..6af336637b 100644 --- a/integration_test/mounts_test.go +++ b/integration_test/mounts_test.go @@ -105,7 +105,11 @@ func (s *mountsSuite) Test_MountWrite() { } ctx := s.Context() - s.ApplyApp(ctx, "hello-w-volumes", "deploy/hello") + k8s := filepath.Join("testdata", "k8s") + s.ApplyTemplate(ctx, filepath.Join(k8s, "hello-pv-volume.goyaml"), &itest.PersistentVolume{ + Name: "hello", + MountDirectory: "/data", + }) defer s.DeleteSvcAndWorkload(ctx, "deploy", "hello") mountPoint := filepath.Join(s.T().TempDir(), "mnt") @@ -123,8 +127,8 @@ func (s *mountsSuite) Test_MountWrite() { mountPoint = filepath.Join(s.T().TempDir(), "data") itest.TelepresenceOk(ctx, "intercept", "hello", "--mount", mountPoint, "--port", "80:80") defer itest.TelepresenceOk(ctx, "leave", "hello") + s.CapturePodLogs(ctx, "hello", "traffic-agent", s.AppNamespace()) - time.Sleep(2 * time.Second) path = filepath.Join(mountPoint, "data", "hello.txt") data, err := os.ReadFile(path) rq.NoError(err) diff --git a/integration_test/testdata/k8s/hello-pv-volume.goyaml b/integration_test/testdata/k8s/hello-pv-volume.goyaml new file mode 100644 index 0000000000..ce3c8c9c89 --- /dev/null +++ b/integration_test/testdata/k8s/hello-pv-volume.goyaml @@ -0,0 +1,74 @@ +{{- /*gotype: github.com/telepresenceio/telepresence/v2/integration_test/itest.PersistentVolume*/ -}} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: local-pv + labels: + purpose: tp-cli-testing +spec: + capacity: + storage: 0.5Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /data/local-pv +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: local-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Name }} +spec: + type: ClusterIP + selector: + app: {{ .Name }} + ports: + - name: proxied + port: 80 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Name }} + labels: + app: {{ .Name }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Name }} + template: + metadata: + labels: + app: {{ .Name }} + spec: + volumes: + - name: rw-volume + persistentVolumeClaim: + claimName: local-pvc + containers: + - name: {{ .Name }}-app + image: ghcr.io/telepresenceio/echo-server:latest + ports: + - containerPort: 8080 + name: http + resources: + limits: + cpu: 50m + memory: 128Mi + volumeMounts: + - mountPath: {{ .MountDirectory }} + name: rw-volume diff --git a/integration_test/testdata/k8s/hello-pv-volume.yaml b/integration_test/testdata/k8s/hello-pv-volume.yaml deleted file mode 100644 index 83ab70cd5f..0000000000 --- a/integration_test/testdata/k8s/hello-pv-volume.yaml +++ /dev/null @@ -1,75 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: {{ .Name }}-{{ .Namespace }}-volume - labels: - type: local -spec: - storageClassName: manual - capacity: - storage: {{ .VolumeSize }} - accessModes: - - ReadWriteMany - hostPath: - path: "/mnt/{{ .Name }}-{{ .Namespace }}" ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ .Name }}-claim -spec: - volumeName: {{ .Name }}-{{ .Namespace }}-volume - storageClassName: manual - accessModes: - - ReadWriteMany - resources: - requests: - storage: {{ .VolumeSize }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ .Name }} -spec: - type: ClusterIP - selector: - app: {{ .Name }} - ports: - - name: proxied - port: 80 - targetPort: http ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Name }} - labels: - app: {{ .Name }} -spec: - replicas: 1 - selector: - matchLabels: - app: {{ .Name }} - template: - metadata: - labels: - app: {{ .Name }} - spec: - volumes: - - name: scratch-volume - persistentVolumeClaim: - claimName: {{ .Name }}-claim - containers: - - name: echo-easy - image: ghcr.io/telepresenceio/echo-server:latest - ports: - - containerPort: 8080 - name: http - resources: - limits: - cpu: 50m - memory: 128Mi - volumeMounts: - - mountPath: "/home/scratch" - name: scratch-volume diff --git a/pkg/client/userd/trafficmgr/ingest.go b/pkg/client/userd/trafficmgr/ingest.go index 2aac4a7220..b1ea55cbcb 100644 --- a/pkg/client/userd/trafficmgr/ingest.go +++ b/pkg/client/userd/trafficmgr/ingest.go @@ -3,6 +3,7 @@ package trafficmgr import ( "context" "fmt" + "sync" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -26,6 +27,7 @@ func (ik ingestKey) String() string { type ingest struct { *manager.AgentInfo ingestKey + wg sync.WaitGroup ctx context.Context cancel context.CancelFunc localMountPoint string @@ -51,6 +53,7 @@ func (ig *ingest) podAccess(rd daemon.DaemonClient) *podAccess { localMountPort: ig.localMountPort, mounter: &ig.mounter, readOnly: true, + wg: &ig.wg, } if err := pa.ensureAccess(ig.ctx, rd); err != nil { dlog.Error(ig.ctx, err) @@ -250,5 +253,6 @@ func (s *session) LeaveIngest(c context.Context, rq *rpc.IngestIdentifier) (ii * } s.stopHandler(c, fmt.Sprintf("%s/%s", ig.workload, ig.container), ig.handlerContainer, ig.pid) ig.cancel() + ig.wg.Wait() return ig.response(), nil } diff --git a/pkg/client/userd/trafficmgr/intercept.go b/pkg/client/userd/trafficmgr/intercept.go index f3a28e1520..bc40b7501b 100644 --- a/pkg/client/userd/trafficmgr/intercept.go +++ b/pkg/client/userd/trafficmgr/intercept.go @@ -69,6 +69,10 @@ type intercept struct { // Mount read-only readOnly bool + + // finalRemovalDone is closed when the traffic-manager sends a snapshot that no longer contains + // this intercept. + finalRemovalDone chan struct{} } // interceptResult is what gets written to the awaitIntercept's waitCh channel when the @@ -120,6 +124,7 @@ func (ic *intercept) podAccess(rd daemon.DaemonClient) *podAccess { localMountPort: ic.localMountPort, readOnly: ic.readOnly, mounter: &ic.Mounter, + wg: &ic.wg, } if err := pa.ensureAccess(ic.ctx, rd); err != nil { dlog.Error(ic.ctx, err) @@ -245,7 +250,6 @@ func (s *session) getCurrentInterceptInfos() []*manager.InterceptInfo { func (s *session) setCurrentIntercepts(ctx context.Context, iis []*manager.InterceptInfo) { s.currentInterceptsLock.Lock() - defer s.currentInterceptsLock.Unlock() intercepts := make(map[string]*intercept, len(iis)) sb := strings.Builder{} sb.WriteByte('[') @@ -256,7 +260,7 @@ func (s *session) setCurrentIntercepts(ctx context.Context, iis []*manager.Inter ii.ClientMountPoint = ic.ClientMountPoint ic.InterceptInfo = ii } else { - ic = &intercept{InterceptInfo: ii} + ic = &intercept{InterceptInfo: ii, finalRemovalDone: make(chan struct{})} ic.ctx, ic.cancel = context.WithCancel(ctx) dlog.Debugf(ctx, "Received new intercept %s", ic.Spec.Name) if aw, ok := s.interceptWaiters[ii.Spec.Name]; ok { @@ -277,14 +281,21 @@ func (s *session) setCurrentIntercepts(ctx context.Context, iis []*manager.Inter dlog.Debugf(ctx, "setCurrentIntercepts(%s)", sb.String()) // Cancel those that no longer exists + var removed []*intercept for id, ic := range s.currentIntercepts { if _, ok := intercepts[id]; !ok { - dlog.Debugf(ctx, "Cancelling context for intercept %s", ic.Spec.Name) - ic.cancel() + removed = append(removed, ic) } } s.currentIntercepts = intercepts s.reconcileAPIServers(ctx) + s.currentInterceptsLock.Unlock() + + for _, ic := range removed { + dlog.Debugf(ctx, "Cancelling context for intercept %s", ic.Spec.Name) + ic.cancel() + close(ic.finalRemovalDone) + } } func InterceptError(tp common.InterceptError, err error) *rpc.InterceptResult { @@ -661,12 +672,22 @@ func (s *session) removeIntercept(c context.Context, ic *intercept) error { ic.wg.Wait() dlog.Debugf(c, "telling manager to remove intercept %s", name) - c, cancel := client.GetConfig(c).Timeouts().TimeoutContext(c, client.TimeoutTrafficManagerAPI) + tos := client.GetConfig(c).Timeouts() + cc, cancel := tos.TimeoutContext(c, client.TimeoutTrafficManagerAPI) defer cancel() - _, err := s.managerClient.RemoveIntercept(c, &manager.RemoveInterceptRequest2{ + _, err := s.managerClient.RemoveIntercept(cc, &manager.RemoveInterceptRequest2{ Session: s.SessionInfo(), Name: name, }) + if err == nil { + select { + case <-c.Done(): + case <-ic.finalRemovalDone: + + // Just in case the traffic-manager dies before it sends a new snapshot to our intercept watcher. + case <-time.After(tos.Get(client.TimeoutTrafficManagerAPI)): + } + } return err } diff --git a/pkg/client/userd/trafficmgr/podaccess.go b/pkg/client/userd/trafficmgr/podaccess.go index effdf65c2d..854df6df6c 100644 --- a/pkg/client/userd/trafficmgr/podaccess.go +++ b/pkg/client/userd/trafficmgr/podaccess.go @@ -27,7 +27,7 @@ type podAccess struct { ctx context.Context // wg is the group to wait for after a call to cancel - wg sync.WaitGroup + wg *sync.WaitGroup localPorts []string workload string @@ -202,7 +202,7 @@ func (lpf *podAccessTracker) privateStart(pa *podAccess) { ctx, cancel := context.WithCancel(pa.ctx) lp := &podAccessSync{workload: pa.workload, cancelPod: cancel} if pa.shouldMount() { - pa.startMount(ctx, &pa.wg, &lp.wg) + pa.startMount(ctx, pa.wg, &lp.wg) } if pa.shouldForward() { pa.startForwards(ctx, &lp.wg) From e08dc78cfb66bda3b801c3a4ca0c5206908283e5 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 1 Feb 2025 13:13:57 +0100 Subject: [PATCH 32/61] Introduce k8sapi.Kind and use it consistently instead of string. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/agent/agent.go | 2 +- cmd/traffic/cmd/manager/cluster/info.go | 1 + .../cmd/manager/managerutil/envconfig.go | 13 +++-- .../cmd/manager/managerutil/envconfig_test.go | 3 +- .../cmd/manager/mutator/agent_injector.go | 19 +------ .../manager/mutator/agent_injector_test.go | 20 ++------ cmd/traffic/cmd/manager/mutator/watcher.go | 31 ++++-------- .../cmd/manager/mutator/workload_watcher.go | 3 ++ cmd/traffic/cmd/manager/service.go | 11 +---- cmd/traffic/cmd/manager/state/intercept.go | 40 ++++++++++++--- .../manager/state/workload_info_watcher.go | 17 +------ pkg/agentconfig/sidecar.go | 4 +- pkg/agentmap/discorvery.go | 37 ++++++++------ pkg/client/userd/trafficmgr/session.go | 27 +++------- pkg/k8sapi/kind.go | 29 +++++++++++ pkg/k8sapi/object.go | 10 ++-- pkg/k8sapi/workload.go | 34 ++++++------- pkg/workload/util.go | 16 ++++++ pkg/workload/watcher.go | 49 ++++++------------- 19 files changed, 172 insertions(+), 194 deletions(-) create mode 100644 pkg/k8sapi/kind.go diff --git a/cmd/traffic/cmd/agent/agent.go b/cmd/traffic/cmd/agent/agent.go index 670a430557..a8d6a4a961 100644 --- a/cmd/traffic/cmd/agent/agent.go +++ b/cmd/traffic/cmd/agent/agent.go @@ -327,7 +327,7 @@ func StartServices(ctx context.Context, g *dgroup.Group, config Config, srv Stat return &rpc.AgentInfo{ Name: ac.AgentName, Namespace: ac.Namespace, - Kind: ac.WorkloadKind, + Kind: string(ac.WorkloadKind), PodName: config.PodName(), PodIp: config.PodIP(), ApiPort: int32(grpcPort), diff --git a/cmd/traffic/cmd/manager/cluster/info.go b/cmd/traffic/cmd/manager/cluster/info.go index 71a050045a..93cc0cbb7f 100644 --- a/cmd/traffic/cmd/manager/cluster/info.go +++ b/cmd/traffic/cmd/manager/cluster/info.go @@ -408,6 +408,7 @@ func (oi *info) ID() string { func (oi *info) ServiceIP() net.IP { return oi.InjectorSvcIp } + func (oi *info) ClusterDomain() string { return oi.Dns.ClusterDomain } diff --git a/cmd/traffic/cmd/manager/managerutil/envconfig.go b/cmd/traffic/cmd/manager/managerutil/envconfig.go index edc7fef9a3..afc4d38af5 100644 --- a/cmd/traffic/cmd/manager/managerutil/envconfig.go +++ b/cmd/traffic/cmd/manager/managerutil/envconfig.go @@ -19,7 +19,6 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" - "github.com/telepresenceio/telepresence/v2/pkg/workload" ) // Env is the traffic-manager's environment. It does not define any defaults because all @@ -71,7 +70,7 @@ type Env struct { ClientDnsIncludeSuffixes []string `env:"CLIENT_DNS_INCLUDE_SUFFIXES, parser=split-trim, default="` ClientConnectionTTL time.Duration `env:"CLIENT_CONNECTION_TTL, parser=time.ParseDuration"` - EnabledWorkloadKinds []workload.Kind `env:"ENABLED_WORKLOAD_KINDS, parser=split-trim, default=Deployment StatefulSet ReplicaSet"` + EnabledWorkloadKinds k8sapi.Kinds `env:"ENABLED_WORKLOAD_KINDS, parser=split-trim, default=Deployment StatefulSet ReplicaSet"` // For testing only CompatibilityVersion *semver.Version `env:"COMPATIBILITY_VERSION, parser=version, default="` @@ -258,24 +257,24 @@ func fieldTypeHandlers() map[reflect.Type]envconfig.FieldTypeHandler { }, Setter: func(dst reflect.Value, src any) { dst.Set(reflect.ValueOf(src.(*semver.Version))) }, } - fhs[reflect.TypeOf([]workload.Kind{})] = envconfig.FieldTypeHandler{ + fhs[reflect.TypeOf(k8sapi.Kinds{})] = envconfig.FieldTypeHandler{ Parsers: map[string]func(string) (any, error){ "split-trim": func(str string) (any, error) { //nolint:unparam // API requirement if len(str) == 0 { return nil, nil } ss := strings.Split(str, " ") - ks := make([]workload.Kind, len(ss)) + ks := make(k8sapi.Kinds, len(ss)) for i, s := range ss { - ks[i] = workload.Kind(s) - if !ks[i].IsValid() { + ks[i] = k8sapi.Kind(s) + if !k8sapi.KnownWorkloadKinds.Contains(ks[i]) { return nil, fmt.Errorf("invalid workload kind: %q", s) } } return ks, nil }, }, - Setter: func(dst reflect.Value, src interface{}) { dst.Set(reflect.ValueOf(src.([]workload.Kind))) }, + Setter: func(dst reflect.Value, src interface{}) { dst.Set(reflect.ValueOf(src.(k8sapi.Kinds))) }, } return fhs } diff --git a/cmd/traffic/cmd/manager/managerutil/envconfig_test.go b/cmd/traffic/cmd/manager/managerutil/envconfig_test.go index 2588f10dd5..ab6f31cc18 100644 --- a/cmd/traffic/cmd/manager/managerutil/envconfig_test.go +++ b/cmd/traffic/cmd/manager/managerutil/envconfig_test.go @@ -13,7 +13,6 @@ import ( "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/managerutil" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" - "github.com/telepresenceio/telepresence/v2/pkg/workload" ) func TestEnvconfig(t *testing.T) { @@ -55,7 +54,7 @@ func TestEnvconfig(t *testing.T) { PodCIDRStrategy: "auto", PodIP: netip.AddrFrom4([4]byte{203, 0, 113, 18}), ServerPort: 8081, - EnabledWorkloadKinds: []workload.Kind{workload.DeploymentKind, workload.StatefulSetKind, workload.ReplicaSetKind}, + EnabledWorkloadKinds: []k8sapi.Kind{k8sapi.DeploymentKind, k8sapi.StatefulSetKind, k8sapi.ReplicaSetKind}, MaxNamespaceSpecificWatchers: 10, } diff --git a/cmd/traffic/cmd/manager/mutator/agent_injector.go b/cmd/traffic/cmd/manager/mutator/agent_injector.go index 3c30d180c8..d736c74c5a 100644 --- a/cmd/traffic/cmd/manager/mutator/agent_injector.go +++ b/cmd/traffic/cmd/manager/mutator/agent_injector.go @@ -27,7 +27,6 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" "github.com/telepresenceio/telepresence/v2/pkg/maps" - "github.com/telepresenceio/telepresence/v2/pkg/workload" ) var podResource = meta.GroupVersionResource{Version: "v1", Group: "", Resource: "pods"} //nolint:gochecknoglobals // constant @@ -138,21 +137,7 @@ func (a *agentInjector) Inject(ctx context.Context, req *admission.AdmissionRequ return nil, nil } - enabledWorkloads := managerutil.GetEnv(ctx).EnabledWorkloadKinds - supportedKinds := make([]string, len(enabledWorkloads)) - for i, wlKind := range enabledWorkloads { - switch wlKind { - case workload.DeploymentKind: - supportedKinds[i] = "Deployment" - case workload.ReplicaSetKind: - supportedKinds[i] = "ReplicaSet" - case workload.StatefulSetKind: - supportedKinds[i] = "StatefulSet" - case workload.RolloutKind: - supportedKinds[i] = "Rollout" - } - } - wl, err := agentmap.FindOwnerWorkload(ctx, k8sapi.Pod(pod), supportedKinds) + wl, err := agentmap.FindOwnerWorkload(ctx, k8sapi.Pod(pod), env.EnabledWorkloadKinds) if err != nil { uwkError := k8sapi.UnsupportedWorkloadKindError("") switch { @@ -639,7 +624,7 @@ func addPodLabels(_ context.Context, pod *core.Pod, config agentconfig.SidecarEx } if _, ok := pod.Labels[agentconfig.WorkloadKindLabel]; !ok { changed = true - lm[agentconfig.WorkloadKindLabel] = config.AgentConfig().WorkloadKind + lm[agentconfig.WorkloadKindLabel] = string(config.AgentConfig().WorkloadKind) } if _, ok := pod.Labels[agentconfig.WorkloadEnabledLabel]; !ok { changed = true diff --git a/cmd/traffic/cmd/manager/mutator/agent_injector_test.go b/cmd/traffic/cmd/manager/mutator/agent_injector_test.go index ba25e036ca..f5adcb8e1d 100644 --- a/cmd/traffic/cmd/manager/mutator/agent_injector_test.go +++ b/cmd/traffic/cmd/manager/mutator/agent_injector_test.go @@ -34,7 +34,6 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/informer" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" "github.com/telepresenceio/telepresence/v2/pkg/labels" - "github.com/telepresenceio/telepresence/v2/pkg/workload" ) const serviceAccountMountPath = "/var/run/secrets/kubernetes.io/serviceaccount" @@ -844,7 +843,7 @@ matchExpressions: AgentPort: 9900, AgentAppProtocolStrategy: appProtoStrategy, - EnabledWorkloadKinds: []workload.Kind{workload.DeploymentKind, workload.StatefulSetKind, workload.ReplicaSetKind}, + EnabledWorkloadKinds: k8sapi.Kinds{k8sapi.DeploymentKind, k8sapi.StatefulSetKind, k8sapi.ReplicaSetKind}, } ctx = managerutil.WithEnv(ctx, env) ctx = setupAgentInjector(t, ctx, clientset) @@ -1986,7 +1985,7 @@ matchExpressions: AgentPort: 9900, AgentInjectPolicy: agentconfig.WhenEnabled, - EnabledWorkloadKinds: []workload.Kind{workload.DeploymentKind, workload.StatefulSetKind, workload.ReplicaSetKind}, + EnabledWorkloadKinds: k8sapi.Kinds{k8sapi.DeploymentKind, k8sapi.StatefulSetKind, k8sapi.ReplicaSetKind}, } ctx = managerutil.WithEnv(ctx, env) if test.envAdditions != nil { @@ -2064,20 +2063,7 @@ func toAdmissionRequest(resource meta.GroupVersionResource, object any) *admissi } func generateForPod(t *testing.T, ctx context.Context, pod *core.Pod, gc agentmap.GeneratorConfig) (agentconfig.SidecarExt, error) { - supportedKinds := make([]string, 0, 4) - for _, wlKind := range managerutil.GetEnv(ctx).EnabledWorkloadKinds { - switch wlKind { - case workload.DeploymentKind: - supportedKinds = append(supportedKinds, "Deployment") - case workload.ReplicaSetKind: - supportedKinds = append(supportedKinds, "ReplicaSet") - case workload.StatefulSetKind: - supportedKinds = append(supportedKinds, "StatefulSet") - case workload.RolloutKind: - supportedKinds = append(supportedKinds, "Rollout") - } - } - wl, err := agentmap.FindOwnerWorkload(ctx, k8sapi.Pod(pod), supportedKinds) + wl, err := agentmap.FindOwnerWorkload(ctx, k8sapi.Pod(pod), managerutil.GetEnv(ctx).EnabledWorkloadKinds) if err != nil { return nil, err } diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index d0d77e0fe5..8533d0ce33 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -117,7 +117,7 @@ func (c *configWatcher) regenerateAgentMaps(ctx context.Context, ns string, gc a kind: ac.WorkloadKind, } newSce, ok := wls[key] - if !ok { + if !ok && managerutil.GetEnv(ctx).EnabledWorkloadKinds.Contains(ac.WorkloadKind) { wl, err := agentmap.GetWorkload(ctx, ac.WorkloadName, ac.Namespace, ac.WorkloadKind) if err != nil { dlog.Errorf(ctx, "unable to load %s %s.%s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace) @@ -131,7 +131,7 @@ func (c *configWatcher) regenerateAgentMaps(ctx context.Context, ns string, gc a wls[key] = newSce c.Store(newSce) } - if !cmp.Equal(newSce, sce, dbpCmp) { + if newSce == nil || !cmp.Equal(newSce, sce, dbpCmp) { go func() { deletePod(ctx, pod) }() @@ -143,7 +143,7 @@ func (c *configWatcher) regenerateAgentMaps(ctx context.Context, ns string, gc a type workloadKey struct { name string namespace string - kind string + kind k8sapi.Kind } const ( @@ -259,13 +259,13 @@ func (c *configWatcher) startInformers(ctx context.Context, ns string) (iwc *inf ifns[serviceWatcher] = c.startServices(ctx, ns) for _, wlKind := range managerutil.GetEnv(ctx).EnabledWorkloadKinds { switch wlKind { - case workload.DeploymentKind: + case k8sapi.DeploymentKind: ifns[deploymentWatcher] = workload.StartDeployments(ctx, ns) - case workload.ReplicaSetKind: + case k8sapi.ReplicaSetKind: ifns[replicaSetWatcher] = workload.StartReplicaSets(ctx, ns) - case workload.StatefulSetKind: + case k8sapi.StatefulSetKind: ifns[statefulSetWatcher] = workload.StartStatefulSets(ctx, ns) - case workload.RolloutKind: + case k8sapi.RolloutKind: ifns[rolloutWatcher] = workload.StartRollouts(ctx, ns) } } @@ -634,7 +634,7 @@ func podIsRunning(pod *core.Pod) bool { } } -func podList(ctx context.Context, kind, name, namespace string) ([]*core.Pod, error) { +func podList(ctx context.Context, kind k8sapi.Kind, name, namespace string) ([]*core.Pod, error) { var lister interface { List(selector labels.Selector) (ret []*core.Pod, err error) } @@ -649,25 +649,12 @@ func podList(ctx context.Context, kind, name, namespace string) ([]*core.Pod, er return nil, fmt.Errorf("error listing pods in namespace %s: %v", namespace, err) } enabledWorkloads := managerutil.GetEnv(ctx).EnabledWorkloadKinds - supportedKinds := make([]string, len(enabledWorkloads)) - for i, wlKind := range enabledWorkloads { - switch wlKind { - case workload.DeploymentKind: - supportedKinds[i] = "Deployment" - case workload.ReplicaSetKind: - supportedKinds[i] = "ReplicaSet" - case workload.StatefulSetKind: - supportedKinds[i] = "StatefulSet" - case workload.RolloutKind: - supportedKinds[i] = "Rollout" - } - } var podsOfInterest []*core.Pod for _, pod := range pods { if !podIsRunning(pod) { continue } - wl, err := agentmap.FindOwnerWorkload(ctx, k8sapi.Pod(pod), supportedKinds) + wl, err := agentmap.FindOwnerWorkload(ctx, k8sapi.Pod(pod), enabledWorkloads) if err == nil { if (kind == "" || wl.GetKind() == kind) && (name == "" || wl.GetName() == name) { podsOfInterest = append(podsOfInterest, pod) diff --git a/cmd/traffic/cmd/manager/mutator/workload_watcher.go b/cmd/traffic/cmd/manager/mutator/workload_watcher.go index ab16ea74be..4785b042e5 100644 --- a/cmd/traffic/cmd/manager/mutator/workload_watcher.go +++ b/cmd/traffic/cmd/manager/mutator/workload_watcher.go @@ -65,6 +65,9 @@ func (c *configWatcher) updateWorkload(ctx context.Context, wl, oldWl k8sapi.Wor switch ia { case "enabled": + if !managerutil.GetEnv(ctx).EnabledWorkloadKinds.Contains(wl.GetKind()) { + return + } img := managerutil.GetAgentImage(ctx) if img == "" { return diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index 699eb3139d..094e4cec52 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -574,16 +574,7 @@ func (s *service) GetKnownWorkloadKinds(ctx context.Context, request *rpc.Sessio enabledWorkloadKinds := managerutil.GetEnv(ctx).EnabledWorkloadKinds kinds := make([]rpc.WorkloadInfo_Kind, len(enabledWorkloadKinds)) for i, wlKind := range enabledWorkloadKinds { - switch wlKind { - case workload.DeploymentKind: - kinds[i] = rpc.WorkloadInfo_DEPLOYMENT - case workload.ReplicaSetKind: - kinds[i] = rpc.WorkloadInfo_REPLICASET - case workload.StatefulSetKind: - kinds[i] = rpc.WorkloadInfo_STATEFULSET - case workload.RolloutKind: - kinds[i] = rpc.WorkloadInfo_ROLLOUT - } + kinds[i] = workload.RpcKind(wlKind) } return &rpc.KnownWorkloadKinds{Kinds: kinds}, nil } diff --git a/cmd/traffic/cmd/manager/state/intercept.go b/cmd/traffic/cmd/manager/state/intercept.go index 7d399f1b51..d8e7d885a6 100644 --- a/cmd/traffic/cmd/manager/state/intercept.go +++ b/cmd/traffic/cmd/manager/state/intercept.go @@ -65,13 +65,37 @@ func (s *state) PrepareIntercept( } spec := cr.InterceptSpec - wl, err := agentmap.GetWorkload(ctx, spec.Agent, spec.Namespace, spec.WorkloadKind) - if err != nil { - if k8sErrors.IsNotFound(err) { - err = errcat.User.New(err) + kind := k8sapi.Kind(spec.WorkloadKind) + enabledWorkloadKinds := managerutil.GetEnv(ctx).EnabledWorkloadKinds + var wl k8sapi.Workload + if kind == "" { + for _, ek := range enabledWorkloadKinds { + wl, err = agentmap.GetWorkload(ctx, spec.Agent, spec.Namespace, ek) + if err == nil { + break + } + if k8sErrors.IsNotFound(err) { + continue + } + dlog.Error(ctx, err) + return interceptError(err) + } + if wl == nil { + // unless there are zero enabled workload kinds, err must be set to a not-found error at this point + return interceptError(errcat.User.New(k8sErrors.NewNotFound(core.Resource("workload"), spec.Agent+"."+spec.Namespace))) + } + } else { + if !enabledWorkloadKinds.Contains(kind) { + return interceptError(errcat.User.Newf("The %s kind is an not enabled workload kind", kind)) + } + wl, err = agentmap.GetWorkload(ctx, spec.Agent, spec.Namespace, kind) + if err != nil { + if k8sErrors.IsNotFound(err) { + err = errcat.User.New(err) + } + dlog.Error(ctx, err) + return interceptError(err) } - dlog.Error(ctx, err) - return interceptError(err) } var rp agentconfig.ReplacePolicy @@ -89,7 +113,7 @@ func (s *state) PrepareIntercept( pi = &rpc.PreparedIntercept{ Namespace: ac.Namespace, AgentImage: ac.AgentImage, - WorkloadKind: ac.WorkloadKind, + WorkloadKind: string(ac.WorkloadKind), ContainerName: spec.ContainerName, ServiceName: spec.ServiceName, } @@ -230,7 +254,7 @@ func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptReques spec := cir.InterceptSpec interceptID := fmt.Sprintf("%s:%s", sessionID, spec.Name) - wl, err := agentmap.GetWorkload(ctx, spec.Agent, spec.Namespace, spec.WorkloadKind) + wl, err := agentmap.GetWorkload(ctx, spec.Agent, spec.Namespace, k8sapi.Kind(spec.WorkloadKind)) if err != nil { code := codes.Internal if k8sErrors.IsNotFound(err) { diff --git a/cmd/traffic/cmd/manager/state/workload_info_watcher.go b/cmd/traffic/cmd/manager/state/workload_info_watcher.go index 78df3a3765..c289d3342e 100644 --- a/cmd/traffic/cmd/manager/state/workload_info_watcher.go +++ b/cmd/traffic/cmd/manager/state/workload_info_watcher.go @@ -156,21 +156,6 @@ func (wf *workloadInfoWatcher) resetTicker() { wf.ticker.Reset(5 * time.Millisecond) } -func rpcKind(s string) rpc.WorkloadInfo_Kind { - switch strings.ToLower(s) { - case "deployment": - return rpc.WorkloadInfo_DEPLOYMENT - case "replicaset": - return rpc.WorkloadInfo_REPLICASET - case "statefulset": - return rpc.WorkloadInfo_STATEFULSET - case "rollout": - return rpc.WorkloadInfo_ROLLOUT - default: - return rpc.WorkloadInfo_UNSPECIFIED - } -} - func rpcWorkloadState(s workload.State) (state rpc.WorkloadInfo_State) { switch s { case workload.StateFailure: @@ -187,7 +172,7 @@ func rpcWorkloadState(s workload.State) (state rpc.WorkloadInfo_State) { func rpcWorkload(wl k8sapi.Workload, as rpc.WorkloadInfo_AgentState, iClients []*rpc.WorkloadInfo_Intercept) *rpc.WorkloadInfo { return &rpc.WorkloadInfo{ - Kind: rpcKind(wl.GetKind()), + Kind: workload.RpcKind(wl.GetKind()), Name: wl.GetName(), Namespace: wl.GetNamespace(), Uid: string(wl.GetUID()), diff --git a/pkg/agentconfig/sidecar.go b/pkg/agentconfig/sidecar.go index 6e2db2fa5a..b80e01ec20 100644 --- a/pkg/agentconfig/sidecar.go +++ b/pkg/agentconfig/sidecar.go @@ -7,6 +7,8 @@ import ( core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/yaml" + + "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" ) const ( @@ -167,7 +169,7 @@ type Sidecar struct { WorkloadName string `json:"workloadName,omitzero"` // The kind of workload that the pod originates from - WorkloadKind string `json:"workloadKind,omitzero"` + WorkloadKind k8sapi.Kind `json:"workloadKind,omitzero"` // The host used when connecting to the traffic-manager ManagerHost string `json:"managerHost,omitzero"` diff --git a/pkg/agentmap/discorvery.go b/pkg/agentmap/discorvery.go index ef47ea2d5a..494324fa0a 100644 --- a/pkg/agentmap/discorvery.go +++ b/pkg/agentmap/discorvery.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "regexp" - "slices" "sort" core "k8s.io/api/core/v1" @@ -24,28 +23,34 @@ import ( var ReplicaSetNameRx = regexp.MustCompile(`\A(.+)-[a-f0-9]+\z`) -func FindOwnerWorkload(ctx context.Context, obj k8sapi.Object, supportedWorkloadKinds []string) (k8sapi.Workload, error) { +func FindOwnerWorkload(ctx context.Context, obj k8sapi.Object, supportedWorkloadKinds k8sapi.Kinds) (k8sapi.Workload, error) { dlog.Tracef(ctx, "FindOwnerWorkload(%s,%s,%s)", obj.GetName(), obj.GetNamespace(), obj.GetKind()) lbs := obj.GetLabels() if wlName, ok := lbs[agentconfig.WorkloadNameLabel]; ok { - return GetWorkload(ctx, wlName, obj.GetNamespace(), lbs[agentconfig.WorkloadKindLabel]) + kind, ok := lbs[agentconfig.WorkloadKindLabel] + if ok && !supportedWorkloadKinds.Contains(k8sapi.Kind(kind)) { + return nil, fmt.Errorf("unable to find %s owner for %s.%s (annotation controlled)", + kind, obj.GetName(), obj.GetNamespace()) + } + return GetWorkload(ctx, wlName, obj.GetNamespace(), k8sapi.Kind(kind)) } refs := obj.GetOwnerReferences() ns := obj.GetNamespace() for i := range refs { if or := &refs[i]; or.Controller != nil && *or.Controller { - if or.Kind == "ReplicaSet" { + kind := k8sapi.Kind(or.Kind) + if kind == k8sapi.ReplicaSetKind && supportedWorkloadKinds.Contains(k8sapi.DeploymentKind) { // Try the common case first. Strip replicaset's generated hash and try to // get the deployment. If this succeeds, we have saved us a replicaset // lookup. if m := ReplicaSetNameRx.FindStringSubmatch(or.Name); m != nil { - if wl, err := GetWorkload(ctx, m[1], ns, "Deployment"); err == nil { + if wl, err := GetWorkload(ctx, m[1], ns, k8sapi.DeploymentKind); err == nil { return wl, nil } } } - if slices.Contains(supportedWorkloadKinds, or.Kind) { - wl, err := GetWorkload(ctx, or.Name, ns, or.Kind) + if supportedWorkloadKinds.Contains(kind) { + wl, err := GetWorkload(ctx, or.Name, ns, kind) if err != nil { return nil, err } @@ -61,7 +66,7 @@ func FindOwnerWorkload(ctx context.Context, obj k8sapi.Object, supportedWorkload return nil, fmt.Errorf("unable to find workload owner for %s.%s", obj.GetName(), obj.GetNamespace()) } -func GetWorkload(ctx context.Context, name, namespace, workloadKind string) (obj k8sapi.Workload, err error) { +func GetWorkload(ctx context.Context, name, namespace string, workloadKind k8sapi.Kind) (obj k8sapi.Workload, err error) { dlog.Tracef(ctx, "GetWorkload(%s,%s,%s)", name, namespace, workloadKind) i := informer.GetFactory(ctx, namespace) if i == nil { @@ -72,18 +77,18 @@ func GetWorkload(ctx context.Context, name, namespace, workloadKind string) (obj return getWorkload(ai, ri, name, namespace, workloadKind) } -func getWorkload(ai apps.Interface, ri argorollouts.RolloutInformer, name, namespace, workloadKind string) (obj k8sapi.Workload, err error) { - switch workloadKind { - case "Deployment": +func getWorkload(ai apps.Interface, ri argorollouts.RolloutInformer, name, namespace string, kind k8sapi.Kind) (obj k8sapi.Workload, err error) { + switch kind { + case k8sapi.DeploymentKind: return getDeployment(ai, name, namespace) - case "ReplicaSet": + case k8sapi.ReplicaSetKind: return getReplicaSet(ai, name, namespace) - case "StatefulSet": + case k8sapi.StatefulSetKind: return getStatefulSet(ai, name, namespace) - case "Rollout": + case k8sapi.RolloutKind: return getRollout(ri, name, namespace) case "": - for _, wk := range []string{"Deployment", "ReplicaSet", "StatefulSet", "Rollout"} { + for _, wk := range k8sapi.KnownWorkloadKinds { if obj, err = getWorkload(ai, ri, name, namespace, wk); err == nil { return obj, nil } @@ -93,7 +98,7 @@ func getWorkload(ai apps.Interface, ri argorollouts.RolloutInformer, name, names } return nil, k8sErrors.NewNotFound(core.Resource("workload"), name+"."+namespace) default: - return nil, k8sapi.UnsupportedWorkloadKindError(workloadKind) + return nil, k8sapi.UnsupportedWorkloadKindError(kind) } } diff --git a/pkg/client/userd/trafficmgr/session.go b/pkg/client/userd/trafficmgr/session.go index 73b3be7b85..6ea2832440 100644 --- a/pkg/client/userd/trafficmgr/session.go +++ b/pkg/client/userd/trafficmgr/session.go @@ -1103,21 +1103,6 @@ func (s *session) eachWorkload(namespaces []string, do func(kind manager.Workloa s.workloadsLock.Unlock() } -func rpcKind(s string) manager.WorkloadInfo_Kind { - switch strings.ToLower(s) { - case "deployment": - return manager.WorkloadInfo_DEPLOYMENT - case "replicaset": - return manager.WorkloadInfo_REPLICASET - case "statefulset": - return manager.WorkloadInfo_STATEFULSET - case "rollout": - return manager.WorkloadInfo_ROLLOUT - default: - return manager.WorkloadInfo_UNSPECIFIED - } -} - func (s *session) localWorkloadsWatcher(ctx context.Context, namespace string, synced *sync.WaitGroup) error { defer func() { if synced != nil { @@ -1146,20 +1131,20 @@ func (s *session) localWorkloadsWatcher(ctx context.Context, namespace string, s fc = informer.GetFactory(ctx, namespace) } - enabledWorkloadKinds := make([]workload.Kind, len(knownWorkloadKinds.Kinds)) + enabledWorkloadKinds := make(k8sapi.Kinds, len(knownWorkloadKinds.Kinds)) for i, kind := range knownWorkloadKinds.Kinds { switch kind { case manager.WorkloadInfo_DEPLOYMENT: - enabledWorkloadKinds[i] = workload.DeploymentKind + enabledWorkloadKinds[i] = k8sapi.DeploymentKind workload.StartDeployments(ctx, namespace) case manager.WorkloadInfo_REPLICASET: - enabledWorkloadKinds[i] = workload.ReplicaSetKind + enabledWorkloadKinds[i] = k8sapi.ReplicaSetKind workload.StartReplicaSets(ctx, namespace) case manager.WorkloadInfo_STATEFULSET: - enabledWorkloadKinds[i] = workload.StatefulSetKind + enabledWorkloadKinds[i] = k8sapi.StatefulSetKind workload.StartStatefulSets(ctx, namespace) case manager.WorkloadInfo_ROLLOUT: - enabledWorkloadKinds[i] = workload.RolloutKind + enabledWorkloadKinds[i] = k8sapi.RolloutKind workload.StartRollouts(ctx, namespace) af := fc.GetArgoRolloutsInformerFactory() af.Start(ctx.Done()) @@ -1192,7 +1177,7 @@ func (s *session) localWorkloadsWatcher(ctx context.Context, namespace string, s } for _, we := range wls { w := we.Workload - key := workloadInfoKey{kind: rpcKind(w.GetKind()), name: w.GetName()} + key := workloadInfoKey{kind: workload.RpcKind(w.GetKind()), name: w.GetName()} if we.Type == workload.EventTypeDelete { delete(workloads, key) } else { diff --git a/pkg/k8sapi/kind.go b/pkg/k8sapi/kind.go new file mode 100644 index 0000000000..bf8298d68b --- /dev/null +++ b/pkg/k8sapi/kind.go @@ -0,0 +1,29 @@ +package k8sapi + +import "slices" + +type Kind string + +const ( + ServiceKind Kind = "Service" + PodKind Kind = "Pod" + DeploymentKind Kind = "Deployment" + StatefulSetKind Kind = "StatefulSet" + ReplicaSetKind Kind = "ReplicaSet" + RolloutKind Kind = "Rollout" +) + +type Kinds []Kind + +func (k Kinds) Contains(kind Kind) bool { + return slices.Contains(k, kind) +} + +var ( + KnownKinds = Kinds{ServiceKind, PodKind, DeploymentKind, StatefulSetKind, ReplicaSetKind, RolloutKind} //nolint:gochecknoglobals // constant + KnownWorkloadKinds = Kinds{DeploymentKind, ReplicaSetKind, StatefulSetKind, RolloutKind} //nolint:gochecknoglobals // constant +) + +func (w Kind) IsValid() bool { + return KnownKinds.Contains(w) +} diff --git a/pkg/k8sapi/object.go b/pkg/k8sapi/object.go index 34da98daf6..f5c75e213b 100644 --- a/pkg/k8sapi/object.go +++ b/pkg/k8sapi/object.go @@ -15,7 +15,7 @@ type Object interface { runtime.Object meta.Object GetAnnotations() map[string]string - GetKind() string + GetKind() Kind Delete(context.Context) error Refresh(context.Context) error Selector() (labels.Selector, error) @@ -105,8 +105,8 @@ func (o *service) ki(c context.Context) typedCore.ServiceInterface { return services(c, o.Namespace) } -func (o *service) GetKind() string { - return "Service" +func (o *service) GetKind() Kind { + return ServiceKind } func (o *service) Delete(c context.Context) error { @@ -156,8 +156,8 @@ func (o *pod) ki(c context.Context) typedCore.PodInterface { return pods(c, o.Namespace) } -func (o *pod) GetKind() string { - return "Pod" +func (o *pod) GetKind() Kind { + return PodKind } func (o *pod) Delete(c context.Context) error { diff --git a/pkg/k8sapi/workload.go b/pkg/k8sapi/workload.go index c4a535b453..97ceeb6ca3 100644 --- a/pkg/k8sapi/workload.go +++ b/pkg/k8sapi/workload.go @@ -25,7 +25,7 @@ type Workload interface { Updated(int64) bool } -type UnsupportedWorkloadKindError string +type UnsupportedWorkloadKindError Kind func (u UnsupportedWorkloadKindError) Error() string { return fmt.Sprintf("unsupported workload kind: %q", string(u)) @@ -40,18 +40,18 @@ func (u UnsupportedWorkloadKindError) Error() string { // 4. Rollouts (Argo Rollouts) // // The first match is returned. -func GetWorkload(c context.Context, name, namespace, workloadKind string) (obj Workload, err error) { - switch workloadKind { - case "Deployment": +func GetWorkload(c context.Context, name, namespace string, kind Kind) (obj Workload, err error) { + switch kind { + case DeploymentKind: obj, err = GetDeployment(c, name, namespace) - case "ReplicaSet": + case ReplicaSetKind: obj, err = GetReplicaSet(c, name, namespace) - case "StatefulSet": + case StatefulSetKind: obj, err = GetStatefulSet(c, name, namespace) - case "Rollout": + case RolloutKind: obj, err = GetRollout(c, name, namespace) case "": - for _, wk := range []string{"Deployment", "ReplicaSet", "StatefulSet", "Rollout"} { + for _, wk := range KnownWorkloadKinds { if obj, err = GetWorkload(c, name, namespace, wk); err == nil { return obj, nil } @@ -61,7 +61,7 @@ func GetWorkload(c context.Context, name, namespace, workloadKind string) (obj W } err = errors2.NewNotFound(core.Resource("workload"), name+"."+namespace) default: - return nil, UnsupportedWorkloadKindError(workloadKind) + return nil, UnsupportedWorkloadKindError(kind) } return obj, err } @@ -243,8 +243,8 @@ func (o *deployment) ki(c context.Context) typedApps.DeploymentInterface { return deployments(c, o.Namespace) } -func (o *deployment) GetKind() string { - return "Deployment" +func (o *deployment) GetKind() Kind { + return DeploymentKind } func (o *deployment) Delete(c context.Context) error { @@ -312,8 +312,8 @@ func (o *rollout) ki(c context.Context) typedArgoRollouts.RolloutInterface { return rollouts(c, o.Namespace) } -func (o *rollout) GetKind() string { - return "Rollout" +func (o *rollout) GetKind() Kind { + return RolloutKind } func (o *rollout) Delete(c context.Context) error { @@ -377,8 +377,8 @@ func (o *replicaSet) ki(c context.Context) typedApps.ReplicaSetInterface { return replicaSets(c, o.Namespace) } -func (o *replicaSet) GetKind() string { - return "ReplicaSet" +func (o *replicaSet) GetKind() Kind { + return ReplicaSetKind } func (o *replicaSet) Delete(c context.Context) error { @@ -442,8 +442,8 @@ func (o *statefulSet) ki(c context.Context) typedApps.StatefulSetInterface { return statefulSets(c, o.Namespace) } -func (o *statefulSet) GetKind() string { - return "StatefulSet" +func (o *statefulSet) GetKind() Kind { + return StatefulSetKind } func (o *statefulSet) Delete(c context.Context) error { diff --git a/pkg/workload/util.go b/pkg/workload/util.go index 21f2a7a27c..865a92fc7e 100644 --- a/pkg/workload/util.go +++ b/pkg/workload/util.go @@ -3,6 +3,7 @@ package workload import ( "k8s.io/apimachinery/pkg/runtime" + "github.com/telepresenceio/telepresence/rpc/v2/manager" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" ) @@ -14,3 +15,18 @@ func FromAny(obj any) (k8sapi.Workload, bool) { } return nil, false } + +func RpcKind(s k8sapi.Kind) manager.WorkloadInfo_Kind { + switch s { + case k8sapi.DeploymentKind: + return manager.WorkloadInfo_DEPLOYMENT + case k8sapi.ReplicaSetKind: + return manager.WorkloadInfo_REPLICASET + case k8sapi.StatefulSetKind: + return manager.WorkloadInfo_STATEFULSET + case k8sapi.RolloutKind: + return manager.WorkloadInfo_ROLLOUT + default: + return manager.WorkloadInfo_UNSPECIFIED + } +} diff --git a/pkg/workload/watcher.go b/pkg/workload/watcher.go index 3eefbd8b88..14b60d4a1e 100644 --- a/pkg/workload/watcher.go +++ b/pkg/workload/watcher.go @@ -36,19 +36,6 @@ type Event struct { Workload k8sapi.Workload } -type Kind string - -const ( - DeploymentKind Kind = "Deployment" - StatefulSetKind Kind = "StatefulSet" - ReplicaSetKind Kind = "ReplicaSet" - RolloutKind Kind = "Rollout" -) - -func (w *Kind) IsValid() bool { - return w != nil && slices.Contains([]Kind{DeploymentKind, StatefulSetKind, ReplicaSetKind, RolloutKind}, *w) -} - func (e EventType) String() string { switch e { case EventTypeAdd: @@ -72,10 +59,10 @@ type watcher struct { subscriptions map[uuid.UUID]chan<- []Event timer *time.Timer events []Event - enabledWorkloadKinds []Kind + enabledWorkloadKinds k8sapi.Kinds } -func NewWatcher(ctx context.Context, ns string, enabledWorkloadKinds []Kind) (Watcher, error) { +func NewWatcher(ctx context.Context, ns string, enabledWorkloadKinds k8sapi.Kinds) (Watcher, error) { w := new(watcher) w.namespace = ns w.enabledWorkloadKinds = enabledWorkloadKinds @@ -107,19 +94,13 @@ func NewWatcher(ctx context.Context, ns string, enabledWorkloadKinds []Kind) (Wa return w, nil } -func hasValidReplicasetOwner(wl k8sapi.Workload, enabledKinds []Kind) bool { +func hasValidReplicasetOwner(wl k8sapi.Workload, enabledKinds k8sapi.Kinds) bool { for _, ref := range wl.GetOwnerReferences() { if ref.Controller != nil && *ref.Controller { - switch ref.Kind { - case "Deployment": - if slices.Contains(enabledKinds, DeploymentKind) { - return true - } - - case "Rollout": - if slices.Contains(enabledKinds, RolloutKind) { - return true - } + kind := k8sapi.Kind(ref.Kind) + switch kind { + case k8sapi.DeploymentKind, k8sapi.RolloutKind: + return enabledKinds.Contains(kind) } } } @@ -133,7 +114,7 @@ func (w *watcher) Subscribe(ctx context.Context) <-chan []Event { kf := informer.GetFactory(ctx, w.namespace) ai := kf.GetK8sInformerFactory().Apps().V1() dlog.Debugf(ctx, "workload.Watcher producing initial events for namespace %s", w.namespace) - if slices.Contains(w.enabledWorkloadKinds, DeploymentKind) { + if w.enabledWorkloadKinds.Contains(k8sapi.DeploymentKind) { if dps, err := ai.Deployments().Lister().Deployments(w.namespace).List(labels.Everything()); err == nil { for _, obj := range dps { if wl, ok := FromAny(obj); ok && !hasValidReplicasetOwner(wl, w.enabledWorkloadKinds) && !agentmap.TrafficManagerSelector.Matches(labels.Set(obj.Labels)) { @@ -145,7 +126,7 @@ func (w *watcher) Subscribe(ctx context.Context) <-chan []Event { } } } - if slices.Contains(w.enabledWorkloadKinds, ReplicaSetKind) { + if w.enabledWorkloadKinds.Contains(k8sapi.ReplicaSetKind) { if rps, err := ai.ReplicaSets().Lister().ReplicaSets(w.namespace).List(labels.Everything()); err == nil { for _, obj := range rps { if wl, ok := FromAny(obj); ok && !hasValidReplicasetOwner(wl, w.enabledWorkloadKinds) { @@ -157,7 +138,7 @@ func (w *watcher) Subscribe(ctx context.Context) <-chan []Event { } } } - if slices.Contains(w.enabledWorkloadKinds, StatefulSetKind) { + if w.enabledWorkloadKinds.Contains(k8sapi.StatefulSetKind) { if sps, err := ai.StatefulSets().Lister().StatefulSets(w.namespace).List(labels.Everything()); err == nil { for _, obj := range sps { if wl, ok := FromAny(obj); ok && !hasValidReplicasetOwner(wl, w.enabledWorkloadKinds) { @@ -169,7 +150,7 @@ func (w *watcher) Subscribe(ctx context.Context) <-chan []Event { } } } - if slices.Contains(w.enabledWorkloadKinds, RolloutKind) { + if w.enabledWorkloadKinds.Contains(k8sapi.RolloutKind) { ri := kf.GetArgoRolloutsInformerFactory().Argoproj().V1alpha1() if sps, err := ri.Rollouts().Lister().Rollouts(w.namespace).List(labels.Everything()); err == nil { for _, obj := range sps { @@ -270,13 +251,13 @@ func (w *watcher) addEventHandler(ctx context.Context, ns string) error { for _, wlKind := range w.enabledWorkloadKinds { var ssi cache.SharedIndexInformer switch wlKind { - case DeploymentKind: + case k8sapi.DeploymentKind: ssi = ai.Deployments().Informer() - case ReplicaSetKind: + case k8sapi.ReplicaSetKind: ssi = ai.ReplicaSets().Informer() - case StatefulSetKind: + case k8sapi.StatefulSetKind: ssi = ai.StatefulSets().Informer() - case RolloutKind: + case k8sapi.RolloutKind: ri := kf.GetArgoRolloutsInformerFactory().Argoproj().V1alpha1() ssi = ri.Rollouts().Informer() default: From 6670491b4633bdd64e1543cfc0eb317176220134 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 1 Feb 2025 14:04:00 +0100 Subject: [PATCH 33/61] Use podUid to identify pods. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/agent/agent.go | 1 + cmd/traffic/cmd/agent/agent_test.go | 5 + cmd/traffic/cmd/agent/config.go | 23 +- .../cmd/manager/mutator/agent_injector.go | 2 +- .../manager/mutator/agent_injector_test.go | 40 + cmd/traffic/cmd/manager/mutator/watcher.go | 32 +- cmd/traffic/cmd/manager/service.go | 3 +- cmd/traffic/cmd/manager/state/intercept.go | 5 +- cmd/traffic/cmd/manager/state/state.go | 12 +- .../manager/state/workload_info_watcher.go | 4 +- .../cmd/manager/test/testdata/agents.yaml | 4 + pkg/agentconfig/container.go | 9 + rpc/manager/manager.pb.go | 1654 +++++++++-------- rpc/manager/manager.proto | 1 + 14 files changed, 944 insertions(+), 851 deletions(-) diff --git a/cmd/traffic/cmd/agent/agent.go b/cmd/traffic/cmd/agent/agent.go index a8d6a4a961..6b904c7e8a 100644 --- a/cmd/traffic/cmd/agent/agent.go +++ b/cmd/traffic/cmd/agent/agent.go @@ -330,6 +330,7 @@ func StartServices(ctx context.Context, g *dgroup.Group, config Config, srv Stat Kind: string(ac.WorkloadKind), PodName: config.PodName(), PodIp: config.PodIP(), + PodUid: string(config.PodUID()), ApiPort: int32(grpcPort), FtpPort: int32(ftpPort), SftpPort: int32(sftpPort), diff --git a/cmd/traffic/cmd/agent/agent_test.go b/cmd/traffic/cmd/agent/agent_test.go index c466f0920d..0e66da924c 100644 --- a/cmd/traffic/cmd/agent/agent_test.go +++ b/cmd/traffic/cmd/agent/agent_test.go @@ -20,6 +20,8 @@ const ( serviceName = "test-echo" namespace = "teltest" podIP = "192.168.50.34" + podName = "test-echo-f4784865-9wgqz" + podUID = "dc6100d6-2316-4eb6-9e99-9bf349877fb8" ) var testConfig = agentconfig.Sidecar{ @@ -63,7 +65,10 @@ func testContext(t *testing.T, env dos.MapEnv) context.Context { cfgJSON, err := agentconfig.MarshalTight(&testConfig) require.NoError(t, err) + env[agentconfig.EnvPrefixAgent+"NAME"] = serviceName env[agentconfig.EnvPrefixAgent+"POD_IP"] = podIP + env[agentconfig.EnvPrefixAgent+"POD_NAME"] = podName + env[agentconfig.EnvPrefixAgent+"POD_UID"] = podUID env[agentconfig.EnvAgentConfig] = cfgJSON ctx := dlog.NewTestContext(t, false) diff --git a/cmd/traffic/cmd/agent/config.go b/cmd/traffic/cmd/agent/config.go index c5aac7ff7a..2caf211f15 100644 --- a/cmd/traffic/cmd/agent/config.go +++ b/cmd/traffic/cmd/agent/config.go @@ -10,6 +10,8 @@ import ( "strconv" "strings" + "k8s.io/apimachinery/pkg/types" + "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/dos" @@ -23,12 +25,14 @@ type Config interface { HasMounts(ctx context.Context) bool PodName() string PodIP() string + PodUID() types.UID } type config struct { sidecarExt agentconfig.SidecarExt podName string podIP string + podUID types.UID } func LoadConfig(ctx context.Context) (Config, error) { @@ -51,8 +55,19 @@ func LoadConfig(ctx context.Context) (Config, error) { if sc.ManagerPort == 0 { sc.ManagerPort = 8081 } - c.podName = dos.Getenv(ctx, "_TEL_AGENT_NAME") - c.podIP = dos.Getenv(ctx, "_TEL_AGENT_POD_IP") + c.podName, ok = dos.LookupEnv(ctx, "_TEL_AGENT_NAME") + if !ok { + return nil, errors.New("missing NAME") + } + c.podIP, ok = dos.LookupEnv(ctx, "_TEL_AGENT_POD_IP") + if !ok { + return nil, errors.New("missing POD_IP") + } + podUID, ok := dos.LookupEnv(ctx, "_TEL_AGENT_POD_UID") + if !ok { + return nil, errors.New("missing POD_UID") + } + c.podUID = types.UID(podUID) for _, cn := range sc.Containers { if err := addAppMounts(ctx, cn); err != nil { return nil, err @@ -61,6 +76,10 @@ func LoadConfig(ctx context.Context) (Config, error) { return &c, nil } +func (c *config) PodUID() types.UID { + return c.podUID +} + func (c *config) HasMounts(ctx context.Context) bool { for _, cn := range c.AgentConfig().Containers { if len(cn.Mounts) > 0 { diff --git a/cmd/traffic/cmd/manager/mutator/agent_injector.go b/cmd/traffic/cmd/manager/mutator/agent_injector.go index d736c74c5a..46cbfbff18 100644 --- a/cmd/traffic/cmd/manager/mutator/agent_injector.go +++ b/cmd/traffic/cmd/manager/mutator/agent_injector.go @@ -109,7 +109,7 @@ func (a *agentInjector) Inject(ctx context.Context, req *admission.AdmissionRequ } if isDelete { - a.agentConfigs.Inactivate(pod.Status.PodIP) + a.agentConfigs.Inactivate(pod.UID) return nil, nil } diff --git a/cmd/traffic/cmd/manager/mutator/agent_injector_test.go b/cmd/traffic/cmd/manager/mutator/agent_injector_test.go index f5adcb8e1d..1914ff28fb 100644 --- a/cmd/traffic/cmd/manager/mutator/agent_injector_test.go +++ b/cmd/traffic/cmd/manager/mutator/agent_injector_test.go @@ -951,6 +951,7 @@ matchExpressions: Annotations: map[string]string{InjectAnnotation: "enabled"}, Labels: map[string]string{"service": name}, OwnerReferences: podOwner(name), + UID: types.UID(uuid.New().String()), } } @@ -1172,6 +1173,11 @@ matchExpressions: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: _TEL_AGENT_POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid - name: _TEL_AGENT_NAME valueFrom: fieldRef: @@ -1266,6 +1272,11 @@ matchExpressions: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: _TEL_AGENT_POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid - name: _TEL_AGENT_NAME valueFrom: fieldRef: @@ -1409,6 +1420,11 @@ matchExpressions: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: _TEL_AGENT_POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid - name: _TEL_AGENT_NAME valueFrom: fieldRef: @@ -1522,6 +1538,11 @@ matchExpressions: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: _TEL_AGENT_POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid - name: _TEL_AGENT_NAME valueFrom: fieldRef: @@ -1634,6 +1655,11 @@ matchExpressions: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: _TEL_AGENT_POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid - name: _TEL_AGENT_NAME valueFrom: fieldRef: @@ -1788,6 +1814,15 @@ matchExpressions: }, }, }, + { + Name: "_TEL_AGENT_POD_UID", + ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.uid", + }, + }, + }, { Name: "_TEL_AGENT_NAME", ValueFrom: &core.EnvVarSource{ @@ -1901,6 +1936,11 @@ matchExpressions: fieldRef: apiVersion: v1 fieldPath: status.podIP + - name: _TEL_AGENT_POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid - name: _TEL_AGENT_NAME valueFrom: fieldRef: diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index 8533d0ce33..49bd996c5c 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -15,6 +15,7 @@ import ( v1 "k8s.io/api/policy/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" "github.com/datawire/dlib/derror" @@ -37,8 +38,8 @@ type Map interface { OnAdd(context.Context, k8sapi.Workload, agentconfig.SidecarExt) error OnDelete(context.Context, string, string) error DeleteMapsAndRolloutAll(context.Context) - IsInactive(podIP string) bool - Inactivate(podIP string) + IsInactive(podID types.UID) bool + Inactivate(podID types.UID) DeletePodsWithConfig(ctx context.Context, wl k8sapi.Workload) error DeletePodsWithConfigMismatch(ctx context.Context, scx agentconfig.SidecarExt) error DeleteAllPodsWithConfig(ctx context.Context, namespace string) error @@ -171,7 +172,7 @@ type configWatcher struct { agentConfigs *xsync.MapOf[string, map[string]agentconfig.SidecarExt] nsLocks *xsync.MapOf[string, *sync.RWMutex] informers *xsync.MapOf[string, *informersWithCancel] - inactivePods *xsync.MapOf[string, inactivation] + inactivePods *xsync.MapOf[types.UID, inactivation] startedAt time.Time terminating atomic.Bool @@ -235,7 +236,7 @@ func NewWatcher() Map { w := &configWatcher{ nsLocks: xsync.NewMapOf[string, *sync.RWMutex](), informers: xsync.NewMapOf[string, *informersWithCancel](), - inactivePods: xsync.NewMapOf[string, inactivation](), + inactivePods: xsync.NewMapOf[types.UID, inactivation](), agentConfigs: xsync.NewMapOf[string, map[string]agentconfig.SidecarExt](), } w.self = w @@ -405,7 +406,7 @@ func (c *configWatcher) startPods(ctx context.Context, ns string) cache.SharedIn } func (c *configWatcher) gcInactivated(now time.Time) { - c.inactivePods.Range(func(key string, value inactivation) bool { + c.inactivePods.Range(func(key types.UID, value inactivation) bool { if now.Sub(value.Time) > time.Minute { c.inactivePods.Delete(key) } @@ -571,8 +572,9 @@ func (c *configWatcher) DeleteAllPodsWithConfig(ctx context.Context, namespace s } func (c *configWatcher) DeleteIfMismatch(ctx context.Context, pod *core.Pod, cfgJSON string) error { - if c.IsDeleted(pod.Name) { - dlog.Tracef(ctx, "Skipping pod %s because it is already deleted", pod.Name) + podID := pod.UID + if c.IsDeleted(podID) { + dlog.Debugf(ctx, "Skipping pod %s because it is already deleted", pod.Name) return nil } a := pod.ObjectMeta.Annotations @@ -585,9 +587,9 @@ func (c *configWatcher) DeleteIfMismatch(ctx context.Context, pod *core.Pod, cfg return nil } var err error - c.inactivePods.Compute(pod.Name, func(v inactivation, loaded bool) (inactivation, bool) { + c.inactivePods.Compute(podID, func(v inactivation, loaded bool) (inactivation, bool) { if loaded && v.deleted { - dlog.Tracef(ctx, "Skipping pod %s because it was deleted by another thread", pod.Name) + dlog.Debugf(ctx, "Skipping pod %s because it was deleted by another goroutine", pod.Name) return v, false } dlog.Debugf(ctx, "Deleting pod %s because its config is no longer valid", pod.Name) @@ -609,19 +611,19 @@ func deletePod(ctx context.Context, pod *core.Pod) { } } -func (c *configWatcher) Inactivate(podName string) { - c.inactivePods.LoadOrCompute(podName, func() inactivation { +func (c *configWatcher) Inactivate(podID types.UID) { + c.inactivePods.LoadOrCompute(podID, func() inactivation { return inactivation{Time: time.Now()} }) } -func (c *configWatcher) IsDeleted(podName string) bool { - v, ok := c.inactivePods.Load(podName) +func (c *configWatcher) IsDeleted(podID types.UID) bool { + v, ok := c.inactivePods.Load(podID) return ok && v.deleted } -func (c *configWatcher) IsInactive(podName string) bool { - _, ok := c.inactivePods.Load(podName) +func (c *configWatcher) IsInactive(podID types.UID) bool { + _, ok := c.inactivePods.Load(podID) return ok } diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index 094e4cec52..120dc5b96c 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -18,6 +18,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" empty "google.golang.org/protobuf/types/known/emptypb" + "k8s.io/apimachinery/pkg/types" "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dgroup" @@ -698,7 +699,7 @@ func (s *service) ReviewIntercept(ctx context.Context, rIReq *rpc.ReviewIntercep if intercept.Spec.Namespace != agent.Namespace || intercept.Spec.Agent != agent.Name { return } - if mutator.GetMap(ctx).IsInactive(agent.PodIp) { + if mutator.GetMap(ctx).IsInactive(types.UID(agent.PodUid)) { dlog.Debugf(ctx, "Pod %s(%s) is blacklisted", agent.PodName, agent.PodIp) return } diff --git a/cmd/traffic/cmd/manager/state/intercept.go b/cmd/traffic/cmd/manager/state/intercept.go index d8e7d885a6..3f93d30767 100644 --- a/cmd/traffic/cmd/manager/state/intercept.go +++ b/cmd/traffic/cmd/manager/state/intercept.go @@ -21,6 +21,7 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dlog" @@ -792,10 +793,10 @@ func (s *state) waitForAgents(ctx context.Context, ac *agentconfig.Sidecar, fail } as := make([]*rpc.AgentInfo, 0, len(snapshot)) for _, a := range snapshot { - if mm.IsInactive(a.PodIp) { + if mm.IsInactive(types.UID(a.PodUid)) { dlog.Debugf(ctx, "Agent %s(%s) is blacklisted", a.PodName, a.PodIp) } else { - dlog.Debugf(ctx, "Agent %s(%s) is ready", a.Name, a.PodIp) + dlog.Debugf(ctx, "Agent %s(%s) is ready", a.PodName, a.PodIp) as = append(as, a) break } diff --git a/cmd/traffic/cmd/manager/state/state.go b/cmd/traffic/cmd/manager/state/state.go index 43e8a172ea..bbde565511 100644 --- a/cmd/traffic/cmd/manager/state/state.go +++ b/cmd/traffic/cmd/manager/state/state.go @@ -315,7 +315,7 @@ func (s *state) RemoveSession(ctx context.Context, sessionID string) { func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *rpc.AgentInfo) { dlog.Debugf(ctx, "Consolidating intercepts after removal of agent %s(%s)", agent.PodName, agent.PodIp) - mutator.GetMap(s.backgroundCtx).Inactivate(agent.PodIp) + mutator.GetMap(s.backgroundCtx).Inactivate(types.UID(agent.PodUid)) s.intercepts.Range(func(interceptID string, intercept *rpc.InterceptInfo) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED || agent.PodIp != intercept.PodIp { // Not of interest. Continue iteration. @@ -447,10 +447,10 @@ func (s *state) CountTunnelEgress() uint64 { // Sessions: Agents //////////////////////////////////////////////////////////////////////////////// func (s *state) AddAgent(ctx context.Context, agent *rpc.AgentInfo, now time.Time) (string, error) { - if mutator.GetMap(ctx).IsInactive(agent.PodIp) { + if mutator.GetMap(ctx).IsInactive(types.UID(agent.PodUid)) { return "", status.Error(codes.Aborted, "inactivated pod") } - sessionID := AgentSessionIDPrefix + agent.PodIp + "." + agent.Namespace + sessionID := AgentSessionIDPrefix + agent.PodUid if oldAgent, hasConflict := s.agents.LoadOrStore(sessionID, agent); hasConflict { return "", status.Error(codes.AlreadyExists, fmt.Sprintf("duplicate id %q, existing %+v, new %+v", sessionID, oldAgent, agent)) } @@ -481,7 +481,7 @@ func (s *state) AddAgent(ctx context.Context, agent *rpc.AgentInfo, now time.Tim func (s *state) GetAgent(sessionID string) *rpc.AgentInfo { if ret, ok := s.agents.Load(sessionID); ok { - if !mutator.GetMap(s.backgroundCtx).IsInactive(ret.PodIp) { + if !mutator.GetMap(s.backgroundCtx).IsInactive(types.UID(ret.PodUid)) { return ret } } @@ -491,7 +491,7 @@ func (s *state) GetAgent(sessionID string) *rpc.AgentInfo { func (s *state) EachAgent(f func(string, *rpc.AgentInfo) bool) { m := mutator.GetMap(s.backgroundCtx) s.agents.Range(func(si string, ag *rpc.AgentInfo) bool { - if !m.IsInactive(ag.PodIp) { + if !m.IsInactive(types.UID(ag.PodUid)) { return f(si, ag) } return true @@ -501,7 +501,7 @@ func (s *state) EachAgent(f func(string, *rpc.AgentInfo) bool) { func (s *state) LoadMatchingAgents(f func(string, *rpc.AgentInfo) bool) map[string]*rpc.AgentInfo { m := mutator.GetMap(s.backgroundCtx) return s.agents.LoadMatching(func(s string, ai *rpc.AgentInfo) bool { - return !m.IsInactive(ai.PodIp) && f(s, ai) + return !m.IsInactive(types.UID(ai.PodUid)) && f(s, ai) }) } diff --git a/cmd/traffic/cmd/manager/state/workload_info_watcher.go b/cmd/traffic/cmd/manager/state/workload_info_watcher.go index c289d3342e..60975be595 100644 --- a/cmd/traffic/cmd/manager/state/workload_info_watcher.go +++ b/cmd/traffic/cmd/manager/state/workload_info_watcher.go @@ -247,7 +247,7 @@ func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[ m := mutator.GetMap(ctx) for k, a := range oldAgentInfos { ai, ok := ais[k] - if !ok || m.IsInactive(ai.PodIp) { + if !ok || m.IsInactive(types.UID(ai.PodUid)) { name := a.Name as := rpc.WorkloadInfo_NO_AGENT_UNSPECIFIED if w, ok := wf.workloadEvents[name]; ok && w.Type != rpc.WorkloadEvent_DELETED { @@ -277,7 +277,7 @@ func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[ } } for _, a := range ais { - if m.IsInactive(a.PodIp) { + if m.IsInactive(types.UID(a.PodUid)) { continue } name := a.Name diff --git a/cmd/traffic/cmd/manager/test/testdata/agents.yaml b/cmd/traffic/cmd/manager/test/testdata/agents.yaml index b122f683fd..110319936a 100644 --- a/cmd/traffic/cmd/manager/test/testdata/agents.yaml +++ b/cmd/traffic/cmd/manager/test/testdata/agents.yaml @@ -3,6 +3,7 @@ hello: name: hello namespace: default pod_name: hello-3487fcx0-uxhgh + pod_uid: e8f38dc9-096d-44dd-9c70-d3d98e18ffe8 hostname: hello-abcdef-123 product: telepresence-agent version: "1" @@ -14,6 +15,7 @@ helloPro: name: hello-pro namespace: default pod_name: hello-pro-f4784865-wgzxg + pod_uid: dc6100d6-2316-4eb6-9e99-9bf349877fb8 hostname: hello-pro-abcdef-123 product: telepresence-agent version: "1" @@ -31,6 +33,7 @@ demo1: name: demo namespace: default pod_name: demo-e2784865-xgamd + pod_uid: 590f13fd-cea5-4c83-9820-c3efa6018ae8 hostname: demo-abcdef-123 product: telepresence-agent version: "1" @@ -48,6 +51,7 @@ demo2: name: demo namespace: default pod_name: demo-33fce865-bgdxf + pod_uid: 12696d21-0da5-4f9a-b403-77e4d6644e2c hostname: demo-abcdef-456 product: telepresence-agent version: "1" diff --git a/pkg/agentconfig/container.go b/pkg/agentconfig/container.go index 4aa99ed10c..c63446a28a 100644 --- a/pkg/agentconfig/container.go +++ b/pkg/agentconfig/container.go @@ -70,6 +70,15 @@ func AgentContainer( }, }, }, + core.EnvVar{ + Name: EnvPrefixAgent + "POD_UID", + ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.uid", + }, + }, + }, core.EnvVar{ Name: EnvPrefixAgent + "NAME", ValueFrom: &core.EnvVarSource{ diff --git a/rpc/manager/manager.pb.go b/rpc/manager/manager.pb.go index b34d6d90c0..e8dccc687b 100644 --- a/rpc/manager/manager.pb.go +++ b/rpc/manager/manager.pb.go @@ -429,6 +429,7 @@ type AgentInfo struct { Namespace string `protobuf:"bytes,7,opt,name=namespace,proto3" json:"namespace,omitempty"` // namespace of the Workload PodName string `protobuf:"bytes,8,opt,name=pod_name,json=podName,proto3" json:"pod_name,omitempty"` // Pod name (from metadata.name) PodIp string `protobuf:"bytes,2,opt,name=pod_ip,json=podIp,proto3" json:"pod_ip,omitempty"` // Pod IP (from status.podIP) + PodUid string `protobuf:"bytes,14,opt,name=pod_uid,json=podUid,proto3" json:"pod_uid,omitempty"` // Pod UID ApiPort int32 `protobuf:"varint,9,opt,name=api_port,json=apiPort,proto3" json:"api_port,omitempty"` // Port number for the agent gRPC API SftpPort int32 `protobuf:"varint,10,opt,name=sftp_port,json=sftpPort,proto3" json:"sftp_port,omitempty"` // Port number for the agent SFTP server FtpPort int32 `protobuf:"varint,11,opt,name=ftp_port,json=ftpPort,proto3" json:"ftp_port,omitempty"` // Port number for the agent FTP server @@ -507,6 +508,13 @@ func (x *AgentInfo) GetPodIp() string { return "" } +func (x *AgentInfo) GetPodUid() string { + if x != nil { + return x.PodUid + } + return "" +} + func (x *AgentInfo) GetApiPort() int32 { if x != nil { return x.ApiPort @@ -4104,7 +4112,7 @@ var file_manager_manager_proto_rawDesc = string([]byte{ 0x6f, 0x64, 0x75, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x22, 0xc4, 0x06, 0x0a, 0x09, 0x41, 0x67, 0x65, + 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x22, 0xdd, 0x06, 0x0a, 0x09, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x1c, @@ -4112,865 +4120,867 @@ var file_manager_manager_proto_rawDesc = string([]byte{ 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x5f, 0x69, - 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x70, 0x12, 0x19, - 0x0a, 0x08, 0x61, 0x70, 0x69, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x07, 0x61, 0x70, 0x69, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x66, 0x74, - 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x66, - 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, - 0x72, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, - 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x49, 0x0a, 0x0a, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, - 0x73, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x65, 0x63, 0x68, 0x61, - 0x6e, 0x69, 0x73, 0x6d, 0x52, 0x0a, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x73, - 0x12, 0x4f, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x0c, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, - 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x73, 0x1a, 0x53, 0x0a, 0x09, 0x4d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0xd2, 0x01, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x60, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, - 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, + 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x70, 0x12, 0x17, + 0x0a, 0x07, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x70, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x69, 0x5f, 0x70, + 0x6f, 0x72, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x61, 0x70, 0x69, 0x50, 0x6f, + 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, + 0x19, 0x0a, 0x08, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x07, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, + 0x6f, 0x64, 0x75, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, + 0x64, 0x75, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x49, + 0x0a, 0x0a, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x52, 0x0a, 0x6d, + 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x73, 0x12, 0x4f, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x43, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x6e, 0x76, - 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x6f, - 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x1a, 0x3e, 0x0a, 0x10, 0x45, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x6c, 0x0a, 0x0f, 0x43, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x43, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, - 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x22, - 0x31, 0x0a, 0x0b, 0x50, 0x6f, 0x72, 0x74, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x12, - 0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x66, 0x72, - 0x6f, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x74, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, - 0x74, 0x6f, 0x22, 0xb8, 0x06, 0x0a, 0x0d, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, - 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x12, 0x14, 0x0a, 0x05, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, - 0x61, 0x64, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, - 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x6e, - 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6d, 0x65, 0x63, - 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, - 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x12, 0x25, 0x0a, 0x0e, 0x6d, 0x65, 0x63, 0x68, 0x61, - 0x6e, 0x69, 0x73, 0x6d, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x0d, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x41, 0x72, 0x67, 0x73, 0x12, 0x1f, - 0x0a, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x48, 0x6f, 0x73, 0x74, 0x12, - 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x12, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0a, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x27, 0x0a, - 0x0f, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x6f, 0x72, 0x74, 0x49, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x6f, - 0x72, 0x74, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x5f, 0x75, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x15, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x18, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, - 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x17, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50, - 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, - 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x50, 0x6f, 0x72, 0x74, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x74, 0x72, 0x69, - 0x70, 0x5f, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x10, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x10, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x74, 0x72, 0x69, 0x70, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, - 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x69, 0x61, 0x6c, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, - 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x64, 0x69, 0x61, 0x6c, 0x54, 0x69, 0x6d, - 0x65, 0x6f, 0x75, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x70, 0x6f, - 0x72, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0a, 0x65, 0x78, 0x74, 0x72, 0x61, - 0x50, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, - 0x18, 0x16, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x12, - 0x26, 0x0a, 0x0f, 0x6e, 0x6f, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x70, 0x6f, - 0x72, 0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x6f, 0x44, 0x65, 0x66, 0x61, - 0x75, 0x6c, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x4a, 0x04, 0x08, 0x0b, 0x10, 0x0c, 0x22, 0x66, 0x0a, - 0x0b, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, - 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, - 0x70, 0x6f, 0x72, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x5f, 0x74, 0x6c, 0x73, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x75, 0x73, 0x65, 0x54, 0x6c, 0x73, 0x12, 0x16, 0x0a, - 0x06, 0x6c, 0x35, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, - 0x35, 0x68, 0x6f, 0x73, 0x74, 0x22, 0xcb, 0x02, 0x0a, 0x0b, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, - 0x77, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3b, 0x0a, 0x07, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x69, 0x6e, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x62, 0x61, - 0x6e, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x64, 0x69, 0x73, 0x70, - 0x6c, 0x61, 0x79, 0x42, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x75, 0x6c, - 0x6c, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x75, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x55, 0x72, 0x6c, 0x12, 0x68, 0x0a, 0x13, 0x61, 0x64, 0x64, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x38, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, - 0x70, 0x65, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x61, 0x64, 0x64, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x1a, 0x44, 0x0a, - 0x16, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x1a, 0x53, 0x0a, 0x09, 0x4d, 0x65, + 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, + 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, + 0x6f, 0x64, 0x75, 0x63, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, + 0xd2, 0x01, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x60, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, + 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, + 0x6f, 0x69, 0x6e, 0x74, 0x1a, 0x3e, 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x6c, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x81, 0x09, 0x0a, 0x0d, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, - 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x63, 0x65, 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x48, - 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, - 0x6b, 0x65, 0x79, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, - 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, 0x6f, 0x6d, - 0x61, 0x69, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x76, 0x69, - 0x65, 0x77, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x44, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x76, - 0x69, 0x65, 0x77, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, - 0x63, 0x52, 0x0b, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x12, 0x50, - 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x63, 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x6f, - 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x6f, - 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x69, 0x5f, 0x70, 0x6f, 0x72, - 0x74, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x61, 0x70, 0x69, 0x50, 0x6f, 0x72, 0x74, - 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x70, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x66, 0x74, 0x70, 0x5f, - 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x66, 0x74, 0x70, - 0x50, 0x6f, 0x72, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, - 0x18, 0x12, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, - 0x2c, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, - 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1f, 0x0a, - 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x10, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2e, - 0x0a, 0x13, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x5f, 0x61, 0x72, 0x67, 0x73, - 0x5f, 0x64, 0x65, 0x73, 0x63, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6d, 0x65, 0x63, - 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x41, 0x72, 0x67, 0x73, 0x44, 0x65, 0x73, 0x63, 0x12, 0x4a, - 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x30, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x4d, 0x0a, 0x08, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, - 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x56, 0x0a, 0x0b, 0x65, 0x6e, 0x76, - 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x11, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, - 0x74, 0x12, 0x3b, 0x0a, 0x0b, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, - 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x1a, 0x3a, - 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x69, 0x72, - 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8d, 0x01, 0x0a, 0x0b, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x10, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x49, 0x6e, 0x73, 0x74, 0x61, - 0x6c, 0x6c, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, - 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, - 0x61, 0x6c, 0x6c, 0x49, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x22, 0x6c, 0x0a, 0x0d, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x43, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x22, 0x31, 0x0a, 0x0b, 0x50, 0x6f, 0x72, 0x74, + 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x74, + 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x74, 0x6f, 0x22, 0xb8, 0x06, 0x0a, 0x0d, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x67, 0x65, + 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x12, + 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6b, 0x69, 0x6e, 0x64, + 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, + 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, + 0x12, 0x25, 0x0a, 0x0e, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x5f, 0x61, 0x72, + 0x67, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, + 0x69, 0x73, 0x6d, 0x41, 0x72, 0x67, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x48, 0x6f, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x64, 0x5f, + 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x64, + 0x50, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x70, + 0x6f, 0x72, 0x74, 0x73, 0x18, 0x12, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6c, 0x6f, 0x63, 0x61, + 0x6c, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x69, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0e, 0x70, 0x6f, 0x72, 0x74, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x12, + 0x2a, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x14, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1f, + 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x0c, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x69, 0x64, 0x12, + 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x15, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x25, + 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x18, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x17, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x2b, 0x0a, + 0x11, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x74, 0x72, 0x69, 0x70, 0x5f, 0x6c, 0x61, 0x74, 0x65, 0x6e, + 0x63, 0x79, 0x18, 0x10, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x74, + 0x72, 0x69, 0x70, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x69, + 0x61, 0x6c, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0b, 0x64, 0x69, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x1f, 0x0a, + 0x0b, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x03, + 0x28, 0x05, 0x52, 0x0a, 0x65, 0x78, 0x74, 0x72, 0x61, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x18, 0x16, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x07, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x6f, 0x5f, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x19, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0d, 0x6e, 0x6f, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x50, 0x6f, 0x72, 0x74, + 0x4a, 0x04, 0x08, 0x0b, 0x10, 0x0c, 0x22, 0x66, 0x0a, 0x0b, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x17, 0x0a, + 0x07, 0x75, 0x73, 0x65, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, + 0x75, 0x73, 0x65, 0x54, 0x6c, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x35, 0x68, 0x6f, 0x73, 0x74, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x35, 0x68, 0x6f, 0x73, 0x74, 0x22, 0xcb, + 0x02, 0x0a, 0x0b, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3b, + 0x0a, 0x07, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x07, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x64, + 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x62, 0x61, 0x6e, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0d, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x42, 0x61, 0x6e, 0x6e, + 0x65, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x75, 0x6c, 0x6c, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x75, + 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x55, 0x72, 0x6c, 0x12, 0x68, 0x0a, 0x13, + 0x61, 0x64, 0x64, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x73, 0x22, 0x4c, 0x0a, 0x11, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, - 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x37, 0x0a, 0x06, 0x61, 0x67, - 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x61, 0x67, 0x65, - 0x6e, 0x74, 0x73, 0x22, 0x5c, 0x0a, 0x15, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x43, 0x0a, 0x0a, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, - 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x73, 0x22, 0xba, 0x01, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, - 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4a, 0x0a, 0x0e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, - 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, - 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x22, 0x65, - 0x0a, 0x12, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xce, 0x03, 0x0a, 0x11, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, - 0x65, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x61, 0x74, 0x65, 0x67, - 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x43, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, - 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, - 0x72, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, - 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x1f, - 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x09, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, - 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x25, 0x0a, 0x0e, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, - 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x64, - 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, - 0x64, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, 0x8b, 0x02, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x51, 0x0a, 0x12, 0x61, 0x64, 0x64, 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, - 0x77, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, - 0x63, 0x48, 0x00, 0x52, 0x10, 0x61, 0x64, 0x64, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x44, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, - 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x13, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x72, - 0x65, 0x76, 0x69, 0x65, 0x77, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x42, 0x17, 0x0a, 0x15, 0x70, - 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x61, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x6a, 0x0a, 0x17, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0x12, - 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x22, 0x66, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x2e, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x41, 0x64, 0x64, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x11, 0x61, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x1a, 0x44, 0x0a, 0x16, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x81, 0x09, 0x0a, + 0x0d, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x37, + 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x53, 0x70, 0x65, + 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x48, 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x0d, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x72, + 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x44, 0x6f, 0x6d, 0x61, 0x69, + 0x6e, 0x12, 0x44, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x73, 0x70, 0x65, + 0x63, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, + 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0b, 0x70, 0x72, 0x65, 0x76, + 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x12, 0x50, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6f, + 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, + 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x64, 0x69, + 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x13, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x19, + 0x0a, 0x08, 0x61, 0x70, 0x69, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x07, 0x61, 0x70, 0x69, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, + 0x5f, 0x69, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x70, + 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0b, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x19, 0x0a, + 0x08, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x12, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x07, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x5f, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x75, 0x6e, + 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x75, + 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x13, 0x6d, 0x65, 0x63, 0x68, 0x61, + 0x6e, 0x69, 0x73, 0x6d, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x18, 0x0c, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x41, + 0x72, 0x67, 0x73, 0x44, 0x65, 0x73, 0x63, 0x12, 0x4a, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xb8, 0x06, 0x0a, 0x16, 0x52, 0x65, 0x76, - 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x50, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x48, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x73, 0x12, 0x4d, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, - 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x15, 0x0a, 0x06, - 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, - 0x64, 0x49, 0x70, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x73, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, - 0x12, 0x19, 0x0a, 0x08, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0c, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x07, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, - 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x13, - 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x64, - 0x65, 0x73, 0x63, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6d, 0x65, 0x63, 0x68, 0x61, - 0x6e, 0x69, 0x73, 0x6d, 0x41, 0x72, 0x67, 0x73, 0x44, 0x65, 0x73, 0x63, 0x12, 0x53, 0x0a, 0x07, - 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x73, 0x12, 0x56, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x09, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, - 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x5f, 0x0a, 0x0b, 0x65, 0x6e, 0x76, - 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x76, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x12, 0x56, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x11, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x0b, 0x6d, 0x6f, + 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x6d, 0x6f, 0x64, + 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x65, 0x0a, 0x0d, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x22, 0x65, 0x0a, 0x0f, 0x4c, 0x6f, - 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, - 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x22, 0x73, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x5f, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x74, 0x72, - 0x61, 0x66, 0x66, 0x69, 0x63, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, - 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x67, - 0x65, 0x6e, 0x74, 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x64, 0x5f, - 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x67, 0x65, 0x74, 0x50, - 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0xb7, 0x02, 0x0a, 0x0c, 0x4c, 0x6f, 0x67, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x6c, - 0x6f, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x6f, - 0x64, 0x4c, 0x6f, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x4c, - 0x6f, 0x67, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x4a, 0x0a, 0x08, - 0x70, 0x6f, 0x64, 0x5f, 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x50, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x07, 0x70, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x1a, 0x3a, 0x0a, 0x0c, 0x50, 0x6f, 0x64, 0x4c, - 0x6f, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3a, 0x0a, 0x0c, 0x50, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x45, + 0x02, 0x38, 0x01, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x29, 0x0a, 0x13, 0x54, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x41, 0x50, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x3c, 0x0a, 0x0c, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x85, 0x01, 0x0a, 0x07, 0x4c, 0x69, - 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, - 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, - 0x6f, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, - 0x64, 0x22, 0x6c, 0x0a, 0x15, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, - 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, - 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, - 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, - 0x72, 0x74, 0x12, 0x1e, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x63, 0x61, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x61, 0x88, - 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x63, 0x61, 0x22, - 0x3c, 0x0a, 0x19, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, - 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, - 0x63, 0x61, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0a, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x22, 0x29, 0x0a, - 0x0d, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, - 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x7c, 0x0a, 0x0b, 0x44, 0x69, 0x61, 0x6c, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x6e, 0x49, 0x64, - 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x74, 0x72, 0x69, 0x70, 0x5f, 0x6c, 0x61, - 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x72, 0x6f, 0x75, - 0x6e, 0x64, 0x74, 0x72, 0x69, 0x70, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x21, 0x0a, - 0x0c, 0x64, 0x69, 0x61, 0x6c, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x0b, 0x64, 0x69, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, - 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x71, 0x0a, 0x0a, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x36, 0x0a, 0x0b, 0x44, 0x4e, 0x53, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x5f, 0x63, 0x6f, - 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, - 0x10, 0x0a, 0x03, 0x72, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x72, 0x72, - 0x73, 0x22, 0xca, 0x01, 0x0a, 0x10, 0x44, 0x4e, 0x53, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, - 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x3d, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2b, - 0x0a, 0x05, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x61, 0x73, 0x6b, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6d, 0x61, 0x73, 0x6b, 0x22, 0x8c, 0x04, 0x0a, 0x0b, - 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x42, 0x0a, 0x0e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, - 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x12, - 0x3c, 0x0a, 0x0b, 0x70, 0x6f, 0x64, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, - 0x74, 0x52, 0x0a, 0x70, 0x6f, 0x64, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, - 0x0e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x6f, - 0x64, 0x49, 0x70, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x70, - 0x6f, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x6f, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, - 0x0f, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x76, 0x63, 0x5f, 0x69, 0x70, - 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x53, 0x76, 0x63, 0x49, 0x70, 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x5f, 0x73, 0x76, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x0f, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x76, 0x63, 0x50, 0x6f, 0x72, - 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x76, - 0x63, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, - 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x76, 0x63, 0x48, 0x6f, 0x73, 0x74, 0x12, 0x37, 0x0a, - 0x07, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x07, 0x72, - 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x2b, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x03, - 0x64, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0b, 0x6b, 0x75, 0x62, 0x65, 0x5f, 0x64, 0x6e, 0x73, 0x5f, - 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6b, 0x75, 0x62, 0x65, 0x44, 0x6e, - 0x73, 0x49, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x64, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0xfa, 0x01, 0x0a, 0x07, 0x52, - 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x12, 0x49, 0x0a, 0x12, 0x61, 0x6c, 0x73, 0x6f, 0x5f, 0x70, - 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, - 0x10, 0x61, 0x6c, 0x73, 0x6f, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, - 0x73, 0x12, 0x4b, 0x0a, 0x13, 0x6e, 0x65, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, - 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x11, 0x6e, 0x65, 0x76, - 0x65, 0x72, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x57, - 0x0a, 0x19, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, - 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x17, - 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x69, 0x6e, 0x67, - 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x22, 0x9b, 0x01, 0x0a, 0x03, 0x44, 0x4e, 0x53, 0x12, - 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x73, 0x75, 0x66, 0x66, 0x69, - 0x78, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x78, - 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x73, 0x75, 0x66, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x53, 0x75, 0x66, - 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x75, 0x62, 0x65, 0x5f, 0x69, 0x70, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6b, 0x75, 0x62, 0x65, 0x49, 0x70, 0x12, 0x25, - 0x0a, 0x0e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0x2c, 0x0a, 0x09, 0x43, 0x4c, 0x49, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x79, 0x61, 0x6d, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x59, - 0x61, 0x6d, 0x6c, 0x22, 0x23, 0x0a, 0x0d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, - 0x65, 0x46, 0x51, 0x4e, 0x12, 0x12, 0x0a, 0x05, 0x66, 0x5f, 0x71, 0x5f, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x66, 0x51, 0x4e, 0x22, 0xc0, 0x01, 0x0a, 0x0c, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x6f, 0x64, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x6f, 0x64, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x70, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x69, - 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x61, 0x70, 0x69, - 0x50, 0x6f, 0x72, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, - 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, - 0x61, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, - 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x52, 0x0a, 0x14, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, - 0x68, 0x6f, 0x74, 0x12, 0x3a, 0x0a, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x1a, 0x3e, 0x0a, 0x10, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x8d, 0x01, 0x0a, 0x0b, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, + 0x2c, 0x0a, 0x12, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, + 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x49, 0x64, 0x12, 0x22, 0x0a, + 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x48, 0x00, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x49, 0x64, 0x88, 0x01, + 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, + 0x22, 0x6c, 0x0a, 0x0d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, + 0x0a, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x22, 0x4c, + 0x0a, 0x11, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x12, 0x37, 0x0a, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x22, - 0x65, 0x0a, 0x12, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x5c, 0x0a, 0x15, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, + 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x43, 0x0a, 0x0a, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, + 0x70, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x73, 0x22, 0xba, 0x01, 0x0a, 0x16, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x29, 0x0a, 0x13, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x22, 0x83, 0x01, 0x0a, 0x0d, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, - 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, - 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x42, - 0x79, 0x74, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x62, - 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x65, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x12, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, - 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x3d, 0x0a, - 0x05, 0x6b, 0x69, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, - 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x05, 0x6b, 0x69, 0x6e, 0x64, 0x73, 0x22, 0x8d, 0x05, 0x0a, - 0x0c, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, - 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x74, 0x65, + 0x6f, 0x6e, 0x12, 0x4a, 0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x5f, + 0x73, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, 0x52, + 0x0d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x17, + 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x22, 0x65, 0x0a, 0x12, 0x45, 0x6e, 0x73, 0x75, 0x72, + 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, + 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xce, + 0x03, 0x0a, 0x11, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x63, 0x65, 0x70, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, + 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, + 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x69, 0x64, + 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, + 0x6f, 0x72, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, + 0x72, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6b, + 0x69, 0x6e, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x6c, + 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, + 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0c, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50, 0x6f, + 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, + 0x0d, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, + 0x8b, 0x02, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, + 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, - 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, - 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x75, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x4e, - 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, - 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x52, 0x0a, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x59, - 0x0a, 0x11, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x5f, 0x63, 0x6c, 0x69, 0x65, - 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x51, 0x0a, 0x12, 0x61, + 0x64, 0x64, 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, + 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, + 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x53, 0x70, 0x65, 0x63, 0x48, 0x00, 0x52, 0x10, 0x61, 0x64, + 0x64, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x34, + 0x0a, 0x15, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, + 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, + 0x13, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x44, 0x6f, + 0x6d, 0x61, 0x69, 0x6e, 0x42, 0x17, 0x0a, 0x15, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x5f, + 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x6a, 0x0a, + 0x17, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x10, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, - 0x70, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3e, 0x0a, 0x05, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x1a, 0x23, 0x0a, 0x09, 0x49, 0x6e, 0x74, - 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x22, 0x55, - 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x45, 0x50, 0x4c, 0x4f, - 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x50, 0x4c, 0x49, - 0x43, 0x41, 0x53, 0x45, 0x54, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x54, 0x41, 0x54, 0x45, - 0x46, 0x55, 0x4c, 0x53, 0x45, 0x54, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x4f, 0x4c, 0x4c, - 0x4f, 0x55, 0x54, 0x10, 0x04, 0x22, 0x4d, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x17, - 0x0a, 0x13, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x56, 0x41, 0x49, 0x4c, - 0x41, 0x42, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, - 0x53, 0x53, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, - 0x52, 0x45, 0x10, 0x03, 0x22, 0x46, 0x0a, 0x0a, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x4e, 0x4f, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x55, - 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, - 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x49, - 0x4e, 0x54, 0x45, 0x52, 0x43, 0x45, 0x50, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0xc7, 0x01, 0x0a, - 0x0d, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3c, - 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x28, 0x2e, 0x74, + 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x66, 0x0a, 0x13, 0x47, 0x65, 0x74, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x22, 0xb8, 0x06, 0x0a, 0x16, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x50, 0x0a, 0x0b, 0x64, 0x69, 0x73, + 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x44, + 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, + 0x64, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x64, 0x49, 0x70, 0x12, 0x1b, 0x0a, 0x09, + 0x73, 0x66, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x08, 0x73, 0x66, 0x74, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x74, 0x70, + 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x66, 0x74, 0x70, + 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, + 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x13, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, + 0x73, 0x6d, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x11, 0x6d, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x69, 0x73, 0x6d, 0x41, 0x72, 0x67, + 0x73, 0x44, 0x65, 0x73, 0x63, 0x12, 0x53, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, + 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, + 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x56, 0x0a, 0x08, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3e, 0x0a, 0x08, - 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, + 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, + 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x12, 0x5f, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, + 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, + 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, + 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x65, 0x0a, 0x0d, + 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, + 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, - 0x66, 0x6f, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x38, 0x0a, 0x04, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x15, 0x0a, 0x11, 0x41, 0x44, 0x44, 0x45, 0x44, 0x5f, 0x55, 0x4e, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4d, - 0x4f, 0x44, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, - 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0x84, 0x01, 0x0a, 0x13, 0x57, 0x6f, 0x72, 0x6b, 0x6c, - 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x30, - 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, - 0x12, 0x3b, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xad, 0x01, - 0x0a, 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x44, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, + 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, + 0x4b, 0x65, 0x79, 0x22, 0x65, 0x0a, 0x0f, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, + 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, + 0x76, 0x65, 0x6c, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x73, 0x0a, 0x0e, 0x47, 0x65, + 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, + 0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x4d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x20, 0x0a, + 0x0c, 0x67, 0x65, 0x74, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0a, 0x67, 0x65, 0x74, 0x50, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x22, + 0xb7, 0x02, 0x0a, 0x0c, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4a, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x6f, 0x64, 0x4c, 0x6f, 0x67, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x17, 0x0a, 0x07, + 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, + 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x4a, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x79, 0x61, 0x6d, + 0x6c, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, + 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x6f, 0x64, 0x59, + 0x61, 0x6d, 0x6c, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x59, 0x61, 0x6d, + 0x6c, 0x1a, 0x3a, 0x0a, 0x0c, 0x50, 0x6f, 0x64, 0x4c, 0x6f, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3a, 0x0a, + 0x0c, 0x50, 0x6f, 0x64, 0x59, 0x61, 0x6d, 0x6c, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x29, 0x0a, 0x13, 0x54, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x41, 0x50, 0x49, 0x49, 0x6e, 0x66, 0x6f, + 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, + 0x70, 0x6f, 0x72, 0x74, 0x22, 0x3c, 0x0a, 0x0c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x22, 0x85, 0x01, 0x0a, 0x07, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x18, + 0x0a, 0x07, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, + 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x65, + 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, + 0x72, 0x4d, 0x73, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x22, 0x6c, 0x0a, 0x15, 0x41, 0x6d, + 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x1e, 0x0a, 0x08, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x63, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, + 0x07, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x61, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, + 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x63, 0x61, 0x22, 0x3c, 0x0a, 0x19, 0x41, 0x6d, 0x62, 0x61, + 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x61, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x63, 0x61, 0x6e, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x22, 0x29, 0x0a, 0x0d, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x22, 0x7c, 0x0a, 0x0b, 0x44, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x17, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x6e, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x6f, 0x75, + 0x6e, 0x64, 0x74, 0x72, 0x69, 0x70, 0x5f, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x74, 0x72, 0x69, 0x70, 0x4c, + 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x69, 0x61, 0x6c, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x64, 0x69, + 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, + 0x71, 0x0a, 0x0a, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, + 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, + 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x22, 0x36, 0x0a, 0x0b, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x05, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x72, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x72, 0x72, 0x73, 0x22, 0xca, 0x01, 0x0a, 0x10, 0x44, + 0x4e, 0x53, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x3b, 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x07, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x30, 0x0a, - 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x12, - 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x76, 0x0a, - 0x16, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x44, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, + 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x08, 0x72, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2b, 0x0a, 0x05, 0x49, 0x50, 0x4e, 0x65, 0x74, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x70, + 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, + 0x6d, 0x61, 0x73, 0x6b, 0x22, 0x8c, 0x04, 0x0a, 0x0b, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x42, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, + 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x12, 0x3c, 0x0a, 0x0b, 0x70, 0x6f, 0x64, 0x5f, + 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, - 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x61, - 0x67, 0x65, 0x6e, 0x74, 0x73, 0x2a, 0xad, 0x01, 0x0a, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, - 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, - 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x01, 0x12, - 0x0b, 0x0a, 0x07, 0x57, 0x41, 0x49, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, - 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, 0x09, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, - 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x41, - 0x47, 0x45, 0x4e, 0x54, 0x10, 0x04, 0x12, 0x10, 0x0a, 0x0c, 0x4e, 0x4f, 0x5f, 0x4d, 0x45, 0x43, - 0x48, 0x41, 0x4e, 0x49, 0x53, 0x4d, 0x10, 0x05, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x50, - 0x4f, 0x52, 0x54, 0x53, 0x10, 0x06, 0x12, 0x0f, 0x0a, 0x0b, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, - 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x07, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x41, 0x44, 0x5f, 0x41, - 0x52, 0x47, 0x53, 0x10, 0x08, 0x32, 0x95, 0x19, 0x0a, 0x07, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x12, 0x45, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, - 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x65, 0x0a, 0x0e, 0x47, 0x65, 0x74, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x2e, 0x74, 0x65, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x0a, 0x70, 0x6f, 0x64, 0x53, + 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x6f, 0x64, 0x49, 0x70, 0x12, 0x28, 0x0a, 0x10, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, + 0x6f, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x5f, 0x73, 0x76, 0x63, 0x5f, 0x69, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0d, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x76, 0x63, 0x49, 0x70, 0x12, 0x2a, + 0x0a, 0x11, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x76, 0x63, 0x5f, 0x70, + 0x6f, 0x72, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x69, 0x6e, 0x6a, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x53, 0x76, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x6e, + 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x76, 0x63, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x18, + 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, + 0x76, 0x63, 0x48, 0x6f, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x07, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, + 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, + 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x07, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x12, + 0x2b, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x03, 0x64, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0b, + 0x6b, 0x75, 0x62, 0x65, 0x5f, 0x64, 0x6e, 0x73, 0x5f, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x09, 0x6b, 0x75, 0x62, 0x65, 0x44, 0x6e, 0x73, 0x49, 0x70, 0x12, 0x25, 0x0a, 0x0e, + 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x22, 0xfa, 0x01, 0x0a, 0x07, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x12, + 0x49, 0x0a, 0x12, 0x61, 0x6c, 0x73, 0x6f, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x73, 0x75, + 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x43, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x69, - 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x19, 0x43, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, - 0x75, 0x64, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x2f, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x10, 0x61, 0x6c, 0x73, 0x6f, 0x50, 0x72, + 0x6f, 0x78, 0x79, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x4b, 0x0a, 0x13, 0x6e, 0x65, + 0x76, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, + 0x50, 0x4e, 0x65, 0x74, 0x52, 0x11, 0x6e, 0x65, 0x76, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x78, 0x79, + 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12, 0x57, 0x0a, 0x19, 0x61, 0x6c, 0x6c, 0x6f, 0x77, + 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x75, 0x62, + 0x6e, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, - 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x0e, 0x47, - 0x65, 0x74, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6d, 0x62, - 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x4a, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x4c, 0x49, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x57, - 0x0a, 0x12, 0x47, 0x65, 0x74, 0x54, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x41, 0x50, 0x49, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x29, 0x2e, 0x74, + 0x72, 0x2e, 0x49, 0x50, 0x4e, 0x65, 0x74, 0x52, 0x17, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x43, 0x6f, + 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, + 0x22, 0x9b, 0x01, 0x0a, 0x03, 0x44, 0x4e, 0x53, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, + 0x75, 0x64, 0x65, 0x5f, 0x73, 0x75, 0x66, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x53, 0x75, 0x66, 0x66, 0x69, + 0x78, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x73, + 0x75, 0x66, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, + 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x17, + 0x0a, 0x07, 0x6b, 0x75, 0x62, 0x65, 0x5f, 0x69, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x06, 0x6b, 0x75, 0x62, 0x65, 0x49, 0x70, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0x2c, + 0x0a, 0x09, 0x43, 0x4c, 0x49, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0x23, 0x0a, 0x0d, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x12, 0x0a, + 0x05, 0x66, 0x5f, 0x71, 0x5f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x66, 0x51, + 0x4e, 0x22, 0xc0, 0x01, 0x0a, 0x0c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, + 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x70, + 0x6f, 0x64, 0x5f, 0x69, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x70, 0x6f, 0x64, + 0x49, 0x70, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x69, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x61, 0x70, 0x69, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x20, 0x0a, + 0x0b, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0b, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, + 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, + 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x52, 0x0a, 0x14, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, + 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x3a, 0x0a, 0x06, + 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x41, 0x50, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x55, 0x0a, 0x0e, 0x41, 0x72, 0x72, 0x69, 0x76, - 0x65, 0x41, 0x73, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x65, 0x0a, 0x12, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, + 0x0a, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, + 0x29, 0x0a, 0x13, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x83, 0x01, 0x0a, 0x0d, 0x54, + 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x2a, 0x0a, 0x11, + 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x67, 0x72, + 0x65, 0x73, 0x73, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0c, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x21, 0x0a, + 0x0c, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x0b, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, 0x42, 0x79, 0x74, 0x65, 0x73, + 0x22, 0x53, 0x0a, 0x12, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, + 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x3d, 0x0a, 0x05, 0x6b, 0x69, 0x6e, 0x64, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, + 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x05, + 0x6b, 0x69, 0x6e, 0x64, 0x73, 0x22, 0x8d, 0x05, 0x0a, 0x0c, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, + 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, + 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b, + 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x4e, 0x0a, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, + 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, + 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x61, 0x67, 0x65, + 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x59, 0x0a, 0x11, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x63, 0x65, 0x70, 0x74, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, + 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x52, 0x10, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x73, 0x12, 0x3e, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, + 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x1a, 0x23, 0x0a, 0x09, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, + 0x16, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x22, 0x55, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, + 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, + 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x45, 0x50, 0x4c, 0x4f, 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x01, + 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x53, 0x45, 0x54, 0x10, 0x02, + 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x54, 0x41, 0x54, 0x45, 0x46, 0x55, 0x4c, 0x53, 0x45, 0x54, 0x10, + 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x10, 0x04, 0x22, 0x4d, + 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, + 0x57, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, + 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x56, 0x41, 0x49, 0x4c, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x01, 0x12, + 0x0f, 0x0a, 0x0b, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x10, 0x02, + 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x03, 0x22, 0x46, 0x0a, + 0x0a, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x4e, + 0x4f, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, + 0x45, 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x43, 0x45, 0x50, + 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0xc7, 0x01, 0x0a, 0x0d, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, + 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, + 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3e, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, + 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x77, 0x6f, 0x72, + 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x38, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x15, 0x0a, + 0x11, 0x41, 0x44, 0x44, 0x45, 0x44, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4d, 0x4f, 0x44, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, + 0x84, 0x01, 0x0a, 0x13, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x53, - 0x0a, 0x0d, 0x41, 0x72, 0x72, 0x69, 0x76, 0x65, 0x41, 0x73, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, - 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, - 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x45, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x23, 0x2e, + 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, + 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xad, 0x01, 0x0a, 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x6c, + 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x44, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, + 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x76, 0x0a, 0x16, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, + 0x61, 0x6c, 0x6c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x44, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x2a, 0xad, + 0x01, 0x0a, 0x18, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x44, 0x69, 0x73, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, + 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x57, 0x41, 0x49, 0x54, + 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, + 0x10, 0x09, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, + 0x03, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x10, 0x04, 0x12, + 0x10, 0x0a, 0x0c, 0x4e, 0x4f, 0x5f, 0x4d, 0x45, 0x43, 0x48, 0x41, 0x4e, 0x49, 0x53, 0x4d, 0x10, + 0x05, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x5f, 0x50, 0x4f, 0x52, 0x54, 0x53, 0x10, 0x06, 0x12, + 0x0f, 0x0a, 0x0b, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x07, + 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x41, 0x44, 0x5f, 0x41, 0x52, 0x47, 0x53, 0x10, 0x08, 0x32, 0x95, + 0x19, 0x0a, 0x07, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x45, 0x0a, 0x07, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x06, 0x44, 0x65, - 0x70, 0x61, 0x72, 0x74, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x4c, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x25, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x32, 0x12, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, + 0x67, 0x65, 0x46, 0x51, 0x4e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x23, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x46, + 0x51, 0x4e, 0x12, 0x65, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, - 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x24, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x0a, 0x47, 0x65, 0x74, + 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, + 0x1d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x12, 0x64, + 0x0a, 0x19, 0x43, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x41, 0x6d, 0x62, 0x61, + 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x12, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x1a, 0x2f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6d, 0x62, 0x61, 0x73, + 0x73, 0x61, 0x64, 0x6f, 0x72, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x6f, 0x75, 0x64, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x50, 0x6f, 0x64, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x2a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, - 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x5b, 0x0a, 0x0b, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, - 0x65, 0x6e, 0x74, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x6d, 0x62, 0x61, 0x73, 0x73, 0x61, 0x64, 0x6f, 0x72, + 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4a, 0x0a, 0x0f, 0x47, + 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x4c, + 0x49, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x57, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x54, 0x65, + 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x41, 0x50, 0x49, 0x12, 0x16, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x41, 0x50, 0x49, 0x49, 0x6e, 0x66, 0x6f, + 0x12, 0x55, 0x0a, 0x0e, 0x41, 0x72, 0x72, 0x69, 0x76, 0x65, 0x41, 0x73, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x30, 0x01, 0x12, 0x5f, 0x0a, 0x0d, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x73, 0x4e, 0x53, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, - 0x74, 0x30, 0x01, 0x12, 0x63, 0x0a, 0x0f, 0x57, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x63, 0x65, 0x70, 0x74, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x6a, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, - 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x12, 0x2b, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, - 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x44, 0x65, 0x6c, - 0x74, 0x61, 0x30, 0x01, 0x12, 0x5a, 0x0a, 0x10, 0x57, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x53, 0x0a, 0x0d, 0x41, 0x72, 0x72, 0x69, 0x76, + 0x65, 0x41, 0x73, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x1f, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, - 0x12, 0x60, 0x0a, 0x0b, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, - 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, - 0x6f, 0x74, 0x12, 0x69, 0x0a, 0x10, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x74, - 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x70, - 0x61, 0x72, 0x65, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x64, 0x0a, - 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x58, 0x0a, 0x0f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, - 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x32, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, - 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, - 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x5e, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, - 0x65, 0x70, 0x74, 0x12, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x57, 0x0a, 0x0f, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, - 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, + 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x45, 0x0a, 0x06, + 0x52, 0x65, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, - 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, 0x15, - 0x47, 0x65, 0x74, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, - 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x06, 0x44, 0x65, 0x70, 0x61, 0x72, 0x74, 0x12, 0x21, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4c, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, + 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, + 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x73, 0x12, 0x24, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, + 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, 0x0a, 0x0e, 0x57, + 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, 0x73, 0x12, 0x21, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x1a, 0x2a, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x64, + 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x5b, + 0x0a, 0x0b, 0x57, 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x21, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x66, + 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x5f, 0x0a, 0x0d, 0x57, + 0x61, 0x74, 0x63, 0x68, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x4e, 0x53, 0x12, 0x23, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, 0x6e, + 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, 0x01, 0x12, 0x63, 0x0a, 0x0f, + 0x57, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x73, 0x12, + 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, + 0x66, 0x6f, 0x1a, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, + 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x30, + 0x01, 0x12, 0x6a, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, + 0x61, 0x64, 0x73, 0x12, 0x2b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, + 0x6f, 0x61, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x30, 0x01, 0x12, 0x5a, 0x0a, + 0x10, 0x57, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, + 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, 0x12, 0x60, 0x0a, 0x0b, 0x45, 0x6e, 0x73, + 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, - 0x64, 0x73, 0x12, 0x50, 0x0a, 0x09, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x12, - 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x16, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x6f, - 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, + 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x69, 0x0a, 0x10, 0x50, + 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, + 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x64, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x58, 0x0a, 0x0f, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, + 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x5e, 0x0a, 0x0c, + 0x47, 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, 0x29, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x57, 0x0a, 0x0f, + 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x12, + 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x65, 0x77, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x64, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x4b, 0x6e, 0x6f, 0x77, + 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, - 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, - 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, - 0x6e, 0x66, 0x6f, 0x1a, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x12, 0x50, 0x0a, 0x0d, 0x57, 0x61, 0x74, 0x63, 0x68, - 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, - 0x1a, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x12, 0x56, 0x0a, 0x06, 0x54, 0x75, 0x6e, - 0x6e, 0x65, 0x6c, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, - 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x57, 0x6f, + 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4b, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x09, 0x4c, + 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x12, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, - 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, - 0x01, 0x12, 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x12, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, - 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, - 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x53, 0x0a, 0x09, 0x57, 0x61, 0x74, 0x63, 0x68, 0x44, 0x69, 0x61, 0x6c, 0x12, 0x21, 0x2e, 0x74, + 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, + 0x16, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, + 0x4e, 0x53, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0e, 0x57, 0x61, 0x74, 0x63, 0x68, + 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x4e, 0x53, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x20, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, - 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x44, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x30, 0x01, 0x12, 0x57, 0x0a, 0x0f, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, - 0x6c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x55, - 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x37, 0x5a, - 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x69, 0x6f, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x32, 0x2f, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x67, 0x65, 0x72, 0x2e, 0x44, 0x4e, 0x53, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, + 0x12, 0x50, 0x0a, 0x0d, 0x57, 0x61, 0x74, 0x63, 0x68, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x25, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x30, 0x01, 0x12, 0x56, 0x0a, 0x06, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x23, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x1a, 0x23, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x0d, 0x52, 0x65, + 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x23, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x2e, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, 0x09, 0x57, 0x61, 0x74, 0x63, + 0x68, 0x44, 0x69, 0x61, 0x6c, 0x12, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, + 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x21, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, + 0x44, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x30, 0x01, 0x12, 0x57, 0x0a, + 0x0f, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, + 0x12, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, + 0x6c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x37, 0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, + 0x65, 0x69, 0x6f, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, + 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x32, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, }) var ( diff --git a/rpc/manager/manager.proto b/rpc/manager/manager.proto index f852193f94..d6908415b8 100644 --- a/rpc/manager/manager.proto +++ b/rpc/manager/manager.proto @@ -31,6 +31,7 @@ message AgentInfo { string namespace = 7; // namespace of the Workload string pod_name = 8; // Pod name (from metadata.name) string pod_ip = 2; // Pod IP (from status.podIP) + string pod_uid = 14; // Pod UID int32 api_port = 9; // Port number for the agent gRPC API int32 sftp_port = 10; // Port number for the agent SFTP server int32 ftp_port = 11; // Port number for the agent FTP server From 6992013b05898c2f9db7f112c28d464d50e75097 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 1 Feb 2025 14:08:17 +0100 Subject: [PATCH 34/61] Merge the traffic-manager interceptStates and intercepts maps. The split between the two was for historical reasons. The `intercepts` map was only capable of storing proto-bufs. Since that's no longer the case, those two maps can be merged. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/manager/mutator/watcher.go | 14 +-- cmd/traffic/cmd/manager/service.go | 18 +-- cmd/traffic/cmd/manager/state/intercept.go | 92 +++++---------- cmd/traffic/cmd/manager/state/state.go | 105 +++++++++--------- cmd/traffic/cmd/manager/state/state_test.go | 3 +- .../manager/state/workload_info_watcher.go | 10 +- 6 files changed, 101 insertions(+), 141 deletions(-) diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index 49bd996c5c..0ef5f0e6a7 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -174,7 +174,7 @@ type configWatcher struct { informers *xsync.MapOf[string, *informersWithCancel] inactivePods *xsync.MapOf[types.UID, inactivation] startedAt time.Time - terminating atomic.Bool + running atomic.Bool self Map // For extension } @@ -304,12 +304,9 @@ func (c *configWatcher) startWatchers(ctx context.Context, iwc *informersWithCan } func (c *configWatcher) StartWatchers(ctx context.Context) error { + defer c.running.Store(true) c.startedAt = time.Now() - ctx, cancel := context.WithCancel(ctx) - c.cancel = func() { - c.terminating.Store(true) - cancel() - } + ctx, c.cancel = context.WithCancel(ctx) var errs []error c.informers.Range(func(ns string, iwc *informersWithCancel) bool { if err := c.startWatchers(ctx, iwc); err != nil { @@ -592,10 +589,7 @@ func (c *configWatcher) DeleteIfMismatch(ctx context.Context, pod *core.Pod, cfg dlog.Debugf(ctx, "Skipping pod %s because it was deleted by another goroutine", pod.Name) return v, false } - dlog.Debugf(ctx, "Deleting pod %s because its config is no longer valid", pod.Name) - go func() { - deletePod(ctx, pod) - }() + deletePod(ctx, pod) return inactivation{Time: time.Now(), deleted: true}, false }) return err diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index 120dc5b96c..df06dcfc99 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -296,7 +296,7 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa agentsCh := s.state.WatchAgents(ctx, func(_ string, info *rpc.AgentInfo) bool { return info.Namespace == ns }) - interceptsCh := s.state.WatchIntercepts(ctx, func(_ string, info *rpc.InterceptInfo) bool { + interceptsCh := s.state.WatchIntercepts(ctx, func(_ string, info *state.Intercept) bool { return info.ClientSession.SessionId == clientSession }) sessionDone, err := s.state.SessionDone(clientSession) @@ -304,7 +304,7 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa return err } - var interceptInfos map[string]*rpc.InterceptInfo + var interceptInfos map[string]*state.Intercept isIntercepted := func(name, namespace string) bool { for _, ii := range interceptInfos { if name == ii.Spec.Agent && namespace == ii.Spec.Namespace { @@ -471,9 +471,9 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W dlog.Debug(ctx, "WatchIntercepts called") var sessionDone <-chan struct{} - var filter func(id string, info *rpc.InterceptInfo) bool + var filter func(id string, info *state.Intercept) bool if sessionID == "" { - filter = func(id string, info *rpc.InterceptInfo) bool { + filter = func(id string, info *state.Intercept) bool { return info.Disposition != rpc.InterceptDispositionType_REMOVED && !state.IsChildIntercept(info.Spec) } } else { @@ -483,7 +483,7 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W } if agent := s.state.GetAgent(sessionID); agent != nil { - filter = func(id string, info *rpc.InterceptInfo) bool { + filter = func(id string, info *state.Intercept) bool { if info.Spec.Namespace != agent.Namespace || info.Spec.Agent != agent.Name { // Don't return intercepts for different agents. return false @@ -509,7 +509,7 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W } } else { // sessionID refers to a client session. - filter = func(id string, info *rpc.InterceptInfo) bool { + filter = func(id string, info *state.Intercept) bool { return info.ClientSession.SessionId == sessionID && info.Disposition != rpc.InterceptDispositionType_REMOVED && !state.IsChildIntercept(info.Spec) @@ -532,7 +532,7 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W dlog.Tracef(ctx, "WatchIntercepts sending update") intercepts := make([]*rpc.InterceptInfo, 0, len(snapshot)) for _, intercept := range snapshot { - intercepts = append(intercepts, intercept) + intercepts = append(intercepts, intercept.InterceptInfo) } resp := &rpc.InterceptInfoSnapshot{ Intercepts: intercepts, @@ -669,7 +669,7 @@ func (s *service) GetIntercept(ctx context.Context, request *rpc.GetInterceptReq return nil, err } if intercept, ok := s.state.GetIntercept(interceptID); ok { - return intercept, nil + return intercept.InterceptInfo, nil } else { return nil, status.Errorf(codes.NotFound, "Intercept named %q not found", request.Name) } @@ -694,7 +694,7 @@ func (s *service) ReviewIntercept(ctx context.Context, rIReq *rpc.ReviewIntercep s.removeExcludedEnvVars(rIReq.Environment) - intercept := s.state.UpdateIntercept(ceptID, func(intercept *rpc.InterceptInfo) { + intercept := s.state.UpdateIntercept(ceptID, func(intercept *state.Intercept) { // Sanity check: The reviewing agent must be an agent for the intercept. if intercept.Spec.Namespace != agent.Namespace || intercept.Spec.Agent != agent.Name { return diff --git a/cmd/traffic/cmd/manager/state/intercept.go b/cmd/traffic/cmd/manager/state/intercept.go index 3f93d30767..5bab3acf81 100644 --- a/cmd/traffic/cmd/manager/state/intercept.go +++ b/cmd/traffic/cmd/manager/state/intercept.go @@ -8,7 +8,6 @@ import ( "regexp" "sort" "strings" - "sync" "time" "google.golang.org/grpc/codes" @@ -211,7 +210,7 @@ func (s *state) preparePorts(ac *agentconfig.Sidecar, cn *agentconfig.Container, } // Validate that there's no port conflict with other intercepts using the same agent. - otherIcs := s.intercepts.LoadMatching(func(s string, info *rpc.InterceptInfo) bool { + otherIcs := s.intercepts.LoadMatching(func(s string, info *Intercept) bool { return info.Disposition == rpc.InterceptDispositionType_ACTIVE && info.Spec.Agent == ac.AgentName && info.Spec.Namespace == ac.Namespace }) @@ -273,7 +272,7 @@ func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptReques return nil, nil, err } - ret, is, err := s.addIntercept(interceptID, cir) + is, err := s.addIntercept(interceptID, cir) if err != nil { return nil, nil, err } @@ -306,16 +305,14 @@ func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptReques pmSpec.Client = fmt.Sprintf("child %s %s %s", pm, spec.Name, spec.Client) pmInterceptID := fmt.Sprintf("%s:%s", sessionID, pmSpec.Name) - _, _, err = s.addIntercept(pmInterceptID, pmCir) + _, err = s.addIntercept(pmInterceptID, pmCir) if err != nil { return nil, nil, err } // Add finalizer to the interceptState of the parent intercept. is.addFinalizer(func(_ context.Context, _ *rpc.InterceptInfo) error { - if _, loaded := s.intercepts.LoadAndDelete(pmInterceptID); loaded { - s.interceptStates.Delete(pmInterceptID) - } + s.intercepts.LoadAndDelete(pmInterceptID) return nil }) } @@ -323,53 +320,52 @@ func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptReques if err != nil { dlog.Errorf(ctx, "Failed to add finalizer for %s: %v", interceptID, err) } - return client, ret, nil + return client, is.InterceptInfo, nil } func IsChildIntercept(spec *rpc.InterceptSpec) bool { return strings.HasPrefix(spec.Client, "child ") } -func (s *state) addIntercept(id string, cir *rpc.CreateInterceptRequest) (*rpc.InterceptInfo, *interceptState, error) { - cept := s.self.NewInterceptInfo(id, cir) +func (s *state) addIntercept(id string, cir *rpc.CreateInterceptRequest) (*Intercept, error) { + is := s.self.NewInterceptInfo(id, cir) // Wrap each potential-state-change in a // // if cept.Disposition == rpc.InterceptDispositionType_WAITING { … } // // so that we don't need to worry about different state-changes stomping on each-other. - if cept.Disposition == rpc.InterceptDispositionType_WAITING { - if errCode, errMsg := s.checkAgentsForIntercept(cept); errCode != 0 { - cept.Disposition = errCode - cept.Message = errMsg + if is.Disposition == rpc.InterceptDispositionType_WAITING { + if errCode, errMsg := s.checkAgentsForIntercept(is); errCode != 0 { + is.Disposition = errCode + is.Message = errMsg } } - if existingValue, hasConflict := s.intercepts.LoadOrStore(id, cept); hasConflict { + if existingValue, hasConflict := s.intercepts.LoadOrStore(id, is); hasConflict { if existingValue.Disposition != rpc.InterceptDispositionType_REMOVED { - return nil, nil, status.Errorf(codes.AlreadyExists, "Intercept named %q already exists", cept.Spec.Name) + return nil, status.Errorf(codes.AlreadyExists, "Intercept named %q already exists", is.Spec.Name) } - s.intercepts.Store(id, cept) + s.intercepts.Store(id, is) } - - is := newInterceptState(id) - s.interceptStates.Store(id, is) - return cept, is, nil + return is, nil } -func (s *state) NewInterceptInfo(interceptID string, ciReq *rpc.CreateInterceptRequest) *rpc.InterceptInfo { - return &rpc.InterceptInfo{ - Spec: ciReq.InterceptSpec, - Disposition: rpc.InterceptDispositionType_WAITING, - Message: "Waiting for Agent approval", - Id: interceptID, - ClientSession: ciReq.Session, - ModifiedAt: timestamppb.Now(), +func (s *state) NewInterceptInfo(interceptID string, ciReq *rpc.CreateInterceptRequest) *Intercept { + return &Intercept{ + InterceptInfo: &rpc.InterceptInfo{ + Spec: ciReq.InterceptSpec, + Disposition: rpc.InterceptDispositionType_WAITING, + Message: "Waiting for Agent approval", + Id: interceptID, + ClientSession: ciReq.Session, + ModifiedAt: timestamppb.Now(), + }, } } func (s *state) AddInterceptFinalizer(interceptID string, finalizer InterceptFinalizer) error { - is, ok := s.interceptStates.Load(interceptID) + is, ok := s.intercepts.Load(interceptID) if !ok { return status.Errorf(codes.NotFound, "no such intercept %s", interceptID) } @@ -380,7 +376,7 @@ func (s *state) AddInterceptFinalizer(interceptID string, finalizer InterceptFin // getAgentsInterceptedByClient returns the session IDs for each agent that are currently // intercepted by the client with the given client session ID. func (s *state) getAgentsInterceptedByClient(clientSessionID string) map[string]*rpc.AgentInfo { - intercepts := s.intercepts.LoadMatching(func(_ string, ii *rpc.InterceptInfo) bool { + intercepts := s.intercepts.LoadMatching(func(_ string, ii *Intercept) bool { return ii.ClientSession.SessionId == clientSessionID }) if len(intercepts) == 0 { @@ -956,37 +952,3 @@ func findContainerIntercept(ac *agentconfig.Sidecar, cn *agentconfig.Container, } return nil, errcat.User.Newf("%s %s.%s has no container port matching %s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace, pi) } - -type InterceptFinalizer func(ctx context.Context, interceptInfo *rpc.InterceptInfo) error - -type interceptState struct { - sync.Mutex - lastInfoCh chan *rpc.InterceptInfo - finalizers []InterceptFinalizer - interceptID string -} - -func newInterceptState(interceptID string) *interceptState { - is := &interceptState{ - lastInfoCh: make(chan *rpc.InterceptInfo), - interceptID: interceptID, - } - return is -} - -func (is *interceptState) addFinalizer(finalizer InterceptFinalizer) { - is.Lock() - defer is.Unlock() - is.finalizers = append(is.finalizers, finalizer) -} - -func (is *interceptState) terminate(ctx context.Context, interceptInfo *rpc.InterceptInfo) { - is.Lock() - defer is.Unlock() - for i := len(is.finalizers) - 1; i >= 0; i-- { - f := is.finalizers[i] - if err := f(ctx, interceptInfo); err != nil { - dlog.Errorf(ctx, "finalizer for intercept %s failed: %v", interceptInfo.Id, err) - } - } -} diff --git a/cmd/traffic/cmd/manager/state/state.go b/cmd/traffic/cmd/manager/state/state.go index bbde565511..0d4a290f54 100644 --- a/cmd/traffic/cmd/manager/state/state.go +++ b/cmd/traffic/cmd/manager/state/state.go @@ -19,6 +19,7 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" + "k8s.io/apimachinery/pkg/types" "github.com/datawire/dlib/dlog" rpc "github.com/telepresenceio/telepresence/rpc/v2/manager" @@ -34,6 +35,33 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/workload" ) +type InterceptFinalizer func(ctx context.Context, interceptInfo *rpc.InterceptInfo) error + +type Intercept struct { + *rpc.InterceptInfo + finalizers []InterceptFinalizer +} + +func (is *Intercept) Clone() *Intercept { + return &Intercept{ + InterceptInfo: proto.Clone(is.InterceptInfo).(*rpc.InterceptInfo), + finalizers: slices.Clone(is.finalizers), + } +} + +func (is *Intercept) addFinalizer(finalizer InterceptFinalizer) { + is.finalizers = append(is.finalizers, finalizer) +} + +func (is *Intercept) terminate(ctx context.Context) { + for i := len(is.finalizers) - 1; i >= 0; i-- { + f := is.finalizers[i] + if err := f(ctx, is.InterceptInfo); err != nil { + dlog.Errorf(ctx, "finalizer for intercept %s failed: %v", is.Id, err) + } + } +} + type State interface { AddAgent(context.Context, *rpc.AgentInfo, time.Time) (string, error) AddClient(*rpc.ClientInfo, time.Time) string @@ -56,21 +84,18 @@ type State interface { GetSession(string) SessionState GetSessionConsumptionMetrics(string) *SessionConsumptionMetrics GetAllSessionConsumptionMetrics() map[string]*SessionConsumptionMetrics - GetIntercept(string) (*rpc.InterceptInfo, bool) + GetIntercept(string) (*Intercept, bool) GetConnectCounter() *prometheus.CounterVec GetConnectActiveStatus() *prometheus.GaugeVec GetInterceptCounter() *prometheus.CounterVec GetInterceptActiveStatus() *prometheus.GaugeVec HasAgent(name, namespace string) bool MarkSession(*rpc.RemainRequest, time.Time) bool - NewInterceptInfo(string, *rpc.CreateInterceptRequest) *rpc.InterceptInfo + NewInterceptInfo(string, *rpc.CreateInterceptRequest) *Intercept PostLookupDNSResponse(context.Context, *rpc.DNSAgentResponse) EnsureAgent(context.Context, string, string) ([]*rpc.AgentInfo, error) PrepareIntercept(context.Context, *rpc.CreateInterceptRequest) (*rpc.PreparedIntercept, error) RemoveIntercept(context.Context, string) - DropIntercept(string) - FinalizeIntercept(ctx context.Context, intercept *rpc.InterceptInfo) - LoadMatchingIntercepts(filter func(string, *rpc.InterceptInfo) bool) map[string]*rpc.InterceptInfo RemoveSession(context.Context, string) SessionDone(string) (<-chan struct{}, error) SetTempLogLevel(context.Context, *rpc.LogLevelRequest) @@ -81,13 +106,13 @@ type State interface { interceptCounterVec *prometheus.CounterVec, interceptStatusGaugeVec *prometheus.GaugeVec) Tunnel(context.Context, tunnel.Stream) error - UpdateIntercept(string, func(*rpc.InterceptInfo)) *rpc.InterceptInfo + UpdateIntercept(string, func(*Intercept)) *Intercept RefreshSessionConsumptionMetrics(sessionID string) ValidateAgentImage(string, bool) error WaitForTempLogLevel(rpc.Manager_WatchLogLevelServer) error WatchAgents(context.Context, func(sessionID string, agent *rpc.AgentInfo) bool) <-chan map[string]*rpc.AgentInfo WatchDial(sessionID string) <-chan *rpc.DialRequest - WatchIntercepts(context.Context, func(sessionID string, intercept *rpc.InterceptInfo) bool) <-chan map[string]*rpc.InterceptInfo + WatchIntercepts(context.Context, func(sessionID string, intercept *Intercept) bool) <-chan map[string]*Intercept WatchWorkloads(ctx context.Context, namespace string) (ch <-chan []workload.Event, err error) WatchLookupDNS(string) <-chan *rpc.DNSRequest ValidateCreateAgent(context.Context, k8sapi.Workload, agentconfig.SidecarExt) error @@ -124,11 +149,10 @@ type state struct { // 7. `cfgMapLocks` access must be concurrency protected // 8. `cachedAgentImage` access must be concurrency protected // 9. `interceptState` must be concurrency protected and updated/deleted in sync with intercepts - intercepts *watchable.Map[string, *rpc.InterceptInfo] // info for intercepts, keyed by intercept id - agents *watchable.Map[string, *rpc.AgentInfo] // info for agent sessions, keyed by session id - clients *xsync.MapOf[string, *rpc.ClientInfo] // info for client sessions, keyed by session id - sessions *xsync.MapOf[string, SessionState] // info for all sessions, keyed by session id - interceptStates *xsync.MapOf[string, *interceptState] + intercepts *watchable.Map[string, *Intercept] // info for intercepts, keyed by intercept id + agents *watchable.Map[string, *rpc.AgentInfo] // info for agent sessions, keyed by session id + clients *xsync.MapOf[string, *rpc.ClientInfo] // info for client sessions, keyed by session id + sessions *xsync.MapOf[string, SessionState] // info for all sessions, keyed by session id timedLogLevel log.TimedLevel llSubs *loglevelSubscribers workloadWatchers *xsync.MapOf[string, workload.Watcher] // workload watchers, created on demand and keyed by namespace @@ -150,8 +174,8 @@ func (s *state) ManagesNamespace(ctx context.Context, ns string) bool { var NewStateFunc = NewState //nolint:gochecknoglobals // extension point -func interceptEqual(a, b *rpc.InterceptInfo) bool { - return proto.Equal(a, b) +func interceptEqual(a, b *Intercept) bool { + return proto.Equal(a.InterceptInfo, b.InterceptInfo) } func agentsEqual(a, b *rpc.AgentInfo) bool { @@ -162,11 +186,10 @@ func NewState(ctx context.Context) State { loglevel := os.Getenv("LOG_LEVEL") s := &state{ backgroundCtx: ctx, - intercepts: watchable.NewMap[string, *rpc.InterceptInfo](interceptEqual, time.Millisecond), + intercepts: watchable.NewMap[string, *Intercept](interceptEqual, time.Millisecond), agents: watchable.NewMap[string, *rpc.AgentInfo](agentsEqual, time.Millisecond), clients: xsync.NewMapOf[string, *rpc.ClientInfo](), sessions: xsync.NewMapOf[string, SessionState](), - interceptStates: xsync.NewMapOf[string, *interceptState](), workloadWatchers: xsync.NewMapOf[string, workload.Watcher](), timedLogLevel: log.NewTimedLevel(loglevel, log.SetLevel), llSubs: newLoglevelSubscribers(), @@ -213,7 +236,7 @@ func (s *state) SetSelf(self State) { // status of all agents that would be relevant to the given intercept spec, and returns whether the // state of those agents would require transitioning to an error state. If everything looks good, // it returns the zero error code (InterceptDispositionType_UNSPECIFIED). -func (s *state) checkAgentsForIntercept(intercept *rpc.InterceptInfo) (errCode rpc.InterceptDispositionType, errMsg string) { +func (s *state) checkAgentsForIntercept(intercept *Intercept) (errCode rpc.InterceptDispositionType, errMsg string) { // Don't overwrite an existing error state switch intercept.Disposition { // non-error states //////////////////////////////////////////////////// @@ -316,7 +339,7 @@ func (s *state) RemoveSession(ctx context.Context, sessionID string) { func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *rpc.AgentInfo) { dlog.Debugf(ctx, "Consolidating intercepts after removal of agent %s(%s)", agent.PodName, agent.PodIp) mutator.GetMap(s.backgroundCtx).Inactivate(types.UID(agent.PodUid)) - s.intercepts.Range(func(interceptID string, intercept *rpc.InterceptInfo) bool { + s.intercepts.Range(func(interceptID string, intercept *Intercept) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED || agent.PodIp != intercept.PodIp { // Not of interest. Continue iteration. return true @@ -325,14 +348,14 @@ func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *rp if errCode, errMsg := s.checkAgentsForIntercept(intercept); errCode != rpc.InterceptDispositionType_UNSPECIFIED { // No agents matching this intercept are available, so the intercept is now dormant or in error. dlog.Debugf(ctx, "Intercept %q no longer has available agents. Setting it disposition to %s", interceptID, errCode) - s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { + s.UpdateIntercept(interceptID, func(intercept *Intercept) { intercept.Disposition = errCode intercept.Message = errMsg }) } else { // The agent is about to die, but apparently more agents are present. Let some other agent pick it up then. dlog.Debugf(ctx, "Intercept %q lost its agent pod %s(%s). Setting it disposition to WAITING", interceptID, agent.PodName, agent.PodIp) - s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { + s.UpdateIntercept(interceptID, func(intercept *Intercept) { intercept.PodIp = "" intercept.PodName = "" intercept.Disposition = rpc.InterceptDispositionType_WAITING @@ -345,7 +368,7 @@ func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *rp func (s *state) gcClientSessionIntercepts(ctx context.Context, sessionID string, client *rpc.ClientInfo) { // GC all intercepts for the client session (intercept.ClientSession.SessionId) - s.intercepts.Range(func(interceptID string, intercept *rpc.InterceptInfo) bool { + s.intercepts.Range(func(interceptID string, intercept *Intercept) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED { return true } @@ -456,7 +479,7 @@ func (s *state) AddAgent(ctx context.Context, agent *rpc.AgentInfo, now time.Tim } s.sessions.Store(sessionID, newAgentSessionState(s.backgroundCtx, now)) - s.intercepts.Range(func(interceptID string, intercept *rpc.InterceptInfo) bool { + s.intercepts.Range(func(interceptID string, intercept *Intercept) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED { return true } @@ -464,12 +487,12 @@ func (s *state) AddAgent(ctx context.Context, agent *rpc.AgentInfo, now time.Tim // because this agent made things inconsistent, or (2) be moved out of a NO_AGENT // state because it just gained an agent. if errCode, errMsg := s.checkAgentsForIntercept(intercept); errCode != 0 { - s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { + s.UpdateIntercept(interceptID, func(intercept *Intercept) { intercept.Disposition = errCode intercept.Message = errMsg }) } else if intercept.Disposition == rpc.InterceptDispositionType_NO_AGENT { - s.UpdateIntercept(interceptID, func(intercept *rpc.InterceptInfo) { + s.UpdateIntercept(interceptID, func(intercept *Intercept) { intercept.Disposition = rpc.InterceptDispositionType_WAITING intercept.Message = "" }) @@ -553,15 +576,15 @@ func (s *state) getAgentsInNamespace(namespace string) map[string]*rpc.AgentInfo // This does not lock; but instead uses CAS and may therefore call the mutator function multiple // times. So: it is safe to perform blocking operations in your mutator function, but you must take // care that it is safe to call your mutator function multiple times. -func (s *state) UpdateIntercept(interceptID string, apply func(*rpc.InterceptInfo)) *rpc.InterceptInfo { +func (s *state) UpdateIntercept(interceptID string, apply func(*Intercept)) *Intercept { for { cur, ok := s.intercepts.Load(interceptID) - if !ok || cur == nil { + if !ok { // Doesn't exist (possibly was deleted while this loop was running). return nil } - newInfo := proto.Clone(cur).(*rpc.InterceptInfo) + newInfo := cur.Clone() apply(newInfo) newInfo.ModifiedAt = timestamppb.Now() @@ -574,25 +597,11 @@ func (s *state) UpdateIntercept(interceptID string, apply func(*rpc.InterceptInf } func (s *state) RemoveIntercept(ctx context.Context, interceptID string) { - if intercept, ok := s.intercepts.LoadAndDelete(interceptID); ok { - s.FinalizeIntercept(ctx, intercept) - } -} - -// FinalizeIntercept calls all the finalizers for the intercept and removes its state. The intercept -// is not removed from the subscription list. -func (s *state) FinalizeIntercept(_ context.Context, intercept *rpc.InterceptInfo) { - if is, ok := s.interceptStates.LoadAndDelete(intercept.Id); ok { - is.terminate(s.backgroundCtx, intercept) + if is, ok := s.intercepts.LoadAndDelete(interceptID); ok { + is.terminate(s.backgroundCtx) } } -// DropIntercept stops tracking intercept with the given ID. It's assume that has been finalized prior to -// this call. -func (s *state) DropIntercept(interceptID string) { - s.intercepts.Delete(interceptID) -} - func (s *state) UninstallAgents(ctx context.Context, ur *rpc.UninstallAgentsRequest) error { sessionID := ur.GetSessionInfo().GetSessionId() clientInfo := s.GetClient(sessionID) @@ -627,18 +636,14 @@ func (s *state) UninstallAgents(ctx context.Context, ur *rpc.UninstallAgentsRequ return nil } -func (s *state) LoadMatchingIntercepts(filter func(string, *rpc.InterceptInfo) bool) map[string]*rpc.InterceptInfo { - return s.intercepts.LoadMatching(filter) -} - -func (s *state) GetIntercept(interceptID string) (*rpc.InterceptInfo, bool) { +func (s *state) GetIntercept(interceptID string) (*Intercept, bool) { return s.intercepts.Load(interceptID) } func (s *state) WatchIntercepts( ctx context.Context, - filter func(sessionID string, intercept *rpc.InterceptInfo) bool, -) <-chan map[string]*rpc.InterceptInfo { + filter func(sessionID string, intercept *Intercept) bool, +) <-chan map[string]*Intercept { return s.intercepts.Subscribe(ctx.Done(), filter) } diff --git a/cmd/traffic/cmd/manager/state/state_test.go b/cmd/traffic/cmd/manager/state/state_test.go index 1c527c7119..7df66e8d1f 100644 --- a/cmd/traffic/cmd/manager/state/state_test.go +++ b/cmd/traffic/cmd/manager/state/state_test.go @@ -30,12 +30,11 @@ func (s *suiteState) SetupTest() { s.ctx = dlog.NewTestContext(s.T(), false) s.state = &state{ backgroundCtx: s.ctx, - intercepts: watchable.NewMap[string, *manager.InterceptInfo](interceptEqual, time.Millisecond), + intercepts: watchable.NewMap[string, *Intercept](interceptEqual, time.Millisecond), agents: watchable.NewMap[string, *manager.AgentInfo](agentsEqual, time.Millisecond), clients: xsync.NewMapOf[string, *manager.ClientInfo](), workloadWatchers: xsync.NewMapOf[string, workload.Watcher](), sessions: xsync.NewMapOf[string, SessionState](), - interceptStates: xsync.NewMapOf[string, *interceptState](), timedLogLevel: log.NewTimedLevel("debug", log.SetLevel), llSubs: newLoglevelSubscribers(), } diff --git a/cmd/traffic/cmd/manager/state/workload_info_watcher.go b/cmd/traffic/cmd/manager/state/workload_info_watcher.go index 60975be595..1d84c48889 100644 --- a/cmd/traffic/cmd/manager/state/workload_info_watcher.go +++ b/cmd/traffic/cmd/manager/state/workload_info_watcher.go @@ -3,12 +3,12 @@ package state import ( "context" "math" - "strings" "time" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "github.com/datawire/dlib/dlog" rpc "github.com/telepresenceio/telepresence/rpc/v2/manager" @@ -30,7 +30,7 @@ type workloadInfoWatcher struct { workloadEvents map[string]*rpc.WorkloadEvent lastEvents map[string]*rpc.WorkloadEvent agentInfos map[string]*rpc.AgentInfo - interceptInfos map[string]*rpc.InterceptInfo + interceptInfos map[string]*Intercept start time.Time ticker *time.Ticker } @@ -72,7 +72,7 @@ func (wf *workloadInfoWatcher) Watch(ctx context.Context, stream rpc.Manager_Wat return info.Namespace == wf.namespace }) - interceptsCh := wf.WatchIntercepts(ctx, func(_ string, info *rpc.InterceptInfo) bool { + interceptsCh := wf.WatchIntercepts(ctx, func(_ string, info *Intercept) bool { return info.Spec.Namespace == wf.namespace }) @@ -304,7 +304,7 @@ func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[ } } -func (wf *workloadInfoWatcher) handleInterceptSnapshot(ctx context.Context, iis map[string]*rpc.InterceptInfo) { +func (wf *workloadInfoWatcher) handleInterceptSnapshot(ctx context.Context, iis map[string]*Intercept) { oldInterceptInfos := wf.interceptInfos wf.interceptInfos = iis for k, ii := range oldInterceptInfos { @@ -324,7 +324,7 @@ func (wf *workloadInfoWatcher) handleInterceptSnapshot(ctx context.Context, iis } } } - ipc := make(map[string][]*rpc.InterceptInfo) + ipc := make(map[string][]*Intercept) for _, ii := range wf.interceptInfos { name := ii.Spec.Agent if ii.Disposition == rpc.InterceptDispositionType_ACTIVE { From e5e8fd5df4b5adbe37910faf4a9cec2f421b7afe Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 1 Feb 2025 14:19:34 +0100 Subject: [PATCH 35/61] Minor fixes to integration tests. Signed-off-by: Thomas Hallgren --- integration_test/cloud_config_test.go | 32 ++++++++++--------- integration_test/intercept_localhost_test.go | 10 ++---- .../workload_configuration_test.go | 2 +- integration_test/workloads_test.go | 1 + 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/integration_test/cloud_config_test.go b/integration_test/cloud_config_test.go index 724577a700..449dae8cbc 100644 --- a/integration_test/cloud_config_test.go +++ b/integration_test/cloud_config_test.go @@ -217,22 +217,24 @@ func (s *notConnectedSuite) Test_RootdCloudLogLevel() { }, 60*time.Second, 5*time.Second, "Root log level not updated in 20 seconds") // Make sure the log level was set back after disconnect - rootLog, err = os.Open(rootLogName) - require.NoError(err) - defer rootLog.Close() - scn = bufio.NewScanner(rootLog) + s.Eventually(func() bool { + rootLog, err = os.Open(rootLogName) + require.NoError(err) + defer rootLog.Close() + scn = bufio.NewScanner(rootLog) - lines = currentLine - currentLine = 0 - for scn.Scan() && currentLine <= lines { - currentLine++ - } + lines = currentLine + currentLine = 0 + for scn.Scan() && currentLine <= lines { + currentLine++ + } - levelSet := false - for scn.Scan() && !levelSet { - levelSet = strings.Contains(scn.Text(), `Logging at this level "info"`) - } - require.True(levelSet, "Root log level not reset after disconnect") + levelSet := false + for scn.Scan() && !levelSet { + levelSet = strings.Contains(scn.Text(), `Logging at this level "info"`) + } + return levelSet + }, 5*time.Second, time.Second, "Root log level not reset after disconnect") // Set it to a "real" value to see that the client-side wins ctx = itest.WithConfig(ctx, func(config client.Config) { @@ -240,7 +242,7 @@ func (s *notConnectedSuite) Test_RootdCloudLogLevel() { }) s.TelepresenceConnect(ctx) itest.TelepresenceDisconnectOk(ctx) - levelSet = false + levelSet := false for scn.Scan() && !levelSet { levelSet = strings.Contains(scn.Text(), `Logging at this level "trace"`) } diff --git a/integration_test/intercept_localhost_test.go b/integration_test/intercept_localhost_test.go index f6a87dad2a..a979efe57a 100644 --- a/integration_test/intercept_localhost_test.go +++ b/integration_test/intercept_localhost_test.go @@ -6,7 +6,6 @@ import ( "net" "net/http" "strconv" - "strings" "time" "github.com/datawire/dlib/dlog" @@ -40,17 +39,10 @@ func (s *interceptLocalhostSuite) SetupSuite() { s.Require().NoError(err) dlog.Infof(ctx, "ip: %s: route: %s", s.defaultRoute.LocalIP, s.defaultRoute) s.port, s.cancelLocal = itest.StartLocalHttpEchoServerWithAddr(ctx, s.ServiceName(), net.JoinHostPort(s.defaultRoute.LocalIP.String(), "0")) - s.CapturePodLogs(ctx, "echo", "traffic-agent", s.AppNamespace()) } func (s *interceptLocalhostSuite) TearDownSuite() { - ctx := s.Context() - itest.TelepresenceOk(ctx, "leave", s.ServiceName()) s.cancelLocal() - s.Eventually(func() bool { - stdout := itest.TelepresenceOk(ctx, "list", "--intercepts") - return !strings.Contains(stdout, s.ServiceName()+": intercepted") - }, 10*time.Second, time.Second) } func (s *interceptLocalhostSuite) TestIntercept_WithCustomLocalhost() { @@ -78,6 +70,8 @@ func (s *interceptLocalhostSuite) TestIntercept_WithCustomLocalhost() { // Run the intercept stdout := itest.TelepresenceOk(ctx, "intercept", s.ServiceName(), "--port", strconv.Itoa(s.port), "--address", s.defaultRoute.LocalIP.String()) + defer itest.TelepresenceOk(ctx, "leave", s.ServiceName()) + s.Require().Contains(stdout, "Using Deployment "+s.ServiceName()) itest.PingInterceptedEchoServer(ctx, s.ServiceName(), "80") } diff --git a/integration_test/workload_configuration_test.go b/integration_test/workload_configuration_test.go index 13c9cfe2f5..1a6c41fb0e 100644 --- a/integration_test/workload_configuration_test.go +++ b/integration_test/workload_configuration_test.go @@ -97,7 +97,7 @@ func (s *workloadConfigurationSuite) Test_InterceptsReplicaSetWithDisabledDeploy interceptableWl := s.KubectlOk(ctx, "get", "replicasets", "-l", fmt.Sprintf("app=%s", wl), "-o", "jsonpath={.items[*].metadata.name}") - s.TelepresenceHelmInstallOK(ctx, true, "--set", "workloads.deployments.enabled=false") + s.TelepresenceHelmInstallOK(ctx, true, "--set", "logLevel=trace", "--set", "workloads.deployments.enabled=false") defer s.TelepresenceHelmInstallOK(ctx, true, "--set", "workloads.deployments.enabled=true") s.TelepresenceConnect(ctx) diff --git a/integration_test/workloads_test.go b/integration_test/workloads_test.go index db9eff6aaa..53dc6cbf87 100644 --- a/integration_test/workloads_test.go +++ b/integration_test/workloads_test.go @@ -24,6 +24,7 @@ func (s *connectedSuite) successfulIntercept(tp, wl, port string) { 2*time.Second, // polling interval ) + itest.TelepresenceOk(ctx, "loglevel", "trace") stdout := itest.TelepresenceOk(ctx, "intercept", "--mount", "false", "--port", port, wl) require.Contains(stdout, "Using "+tp+" "+wl) stdout = itest.TelepresenceOk(ctx, "list", "--intercepts") From 36256b7413a12a79d5e62621badce5b979b3c596 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Fri, 31 Jan 2025 12:13:01 +0100 Subject: [PATCH 36/61] Introduce explicit SessionID type. Also removes the `sessions` map from the traffic-manager's `state` because it has been made redundant. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/agent/client.go | 2 +- cmd/traffic/cmd/agent/fwdstate.go | 4 +- cmd/traffic/cmd/agent/server.go | 9 +- cmd/traffic/cmd/agent/state.go | 8 +- cmd/traffic/cmd/manager/manager.go | 5 +- cmd/traffic/cmd/manager/managerutil/ctx.go | 14 +- cmd/traffic/cmd/manager/service.go | 70 +-- cmd/traffic/cmd/manager/state/consumption.go | 29 +- .../cmd/manager/state/consumption_test.go | 18 +- cmd/traffic/cmd/manager/state/dns.go | 90 ++-- cmd/traffic/cmd/manager/state/intercept.go | 27 +- .../cmd/manager/state/presence_test.go | 11 +- cmd/traffic/cmd/manager/state/session.go | 99 +++-- cmd/traffic/cmd/manager/state/state.go | 407 +++++++++--------- cmd/traffic/cmd/manager/state/state_test.go | 105 ++--- .../manager/state/workload_info_watcher.go | 11 +- pkg/client/agentpf/clients.go | 2 +- pkg/client/remotefs/bridge.go | 4 +- pkg/client/rootd/stream_creator.go | 5 +- pkg/client/userd/trafficmgr/dial_request.go | 2 +- pkg/client/userd/trafficmgr/mount.go | 3 +- pkg/forwarder/tcp.go | 6 +- pkg/forwarder/udp.go | 2 +- pkg/tunnel/client_stream.go | 2 +- pkg/tunnel/dialer.go | 4 +- pkg/tunnel/message.go | 10 +- pkg/tunnel/metrics.go | 14 +- pkg/tunnel/pipe.go | 6 +- pkg/tunnel/stream.go | 8 +- pkg/tunnel/stream_test.go | 4 +- 30 files changed, 473 insertions(+), 508 deletions(-) diff --git a/cmd/traffic/cmd/agent/client.go b/cmd/traffic/cmd/agent/client.go index 6d27829f58..36df189c00 100644 --- a/cmd/traffic/cmd/agent/client.go +++ b/cmd/traffic/cmd/agent/client.go @@ -125,7 +125,7 @@ func TalkToManager(ctx context.Context, address string, info *rpc.AgentInfo, sta return err } wg.Go("dialWait", func(ctx context.Context) error { - return tunnel.DialWaitLoop(ctx, tunnel.ManagerProvider(manager), dialerStream, session.SessionId) + return tunnel.DialWaitLoop(ctx, tunnel.ManagerProvider(manager), dialerStream, tunnel.SessionID(session.SessionId)) }) // Deal with log-level changes diff --git a/cmd/traffic/cmd/agent/fwdstate.go b/cmd/traffic/cmd/agent/fwdstate.go index 2881619c36..2b1ed57db0 100644 --- a/cmd/traffic/cmd/agent/fwdstate.go +++ b/cmd/traffic/cmd/agent/fwdstate.go @@ -63,7 +63,7 @@ func (pm *ProviderMux) ReportMetrics(ctx context.Context, metrics *manager.Tunne pm.AgentProvider.ReportMetrics(ctx, metrics) } -func (pm *ProviderMux) CreateClientStream(ctx context.Context, sessionID string, id tunnel.ConnID, roundTripLatency, dialTimeout time.Duration) (tunnel.Stream, error) { +func (pm *ProviderMux) CreateClientStream(ctx context.Context, sessionID tunnel.SessionID, id tunnel.ConnID, roundTripLatency, dialTimeout time.Duration) (tunnel.Stream, error) { s, err := pm.AgentProvider.CreateClientStream(ctx, sessionID, id, roundTripLatency, dialTimeout) if err == nil && s == nil { s, err = pm.ManagerProvider.CreateClientStream(ctx, sessionID, id, roundTripLatency, dialTimeout) @@ -107,7 +107,7 @@ func (fs *fwdState) HandleIntercepts(ctx context.Context, cepts []*manager.Inter fs.forwarder.SetStreamProvider( &ProviderMux{ AgentProvider: fs, - ManagerProvider: &tunnel.TrafficManagerStreamProvider{Manager: fs.ManagerClient(), AgentSessionID: fs.sessionInfo.SessionId}, + ManagerProvider: &tunnel.TrafficManagerStreamProvider{Manager: fs.ManagerClient(), AgentSessionID: tunnel.SessionID(fs.sessionInfo.SessionId)}, }) } fs.forwarder.SetIntercepting(activeIntercept) diff --git a/cmd/traffic/cmd/agent/server.go b/cmd/traffic/cmd/agent/server.go index 49b90111cb..d9b1cf2b39 100644 --- a/cmd/traffic/cmd/agent/server.go +++ b/cmd/traffic/cmd/agent/server.go @@ -46,7 +46,7 @@ func (s *state) Tunnel(server agent.Agent_TunnelServer) error { <-endPoint.Done() s.ReportMetrics(ctx, &rpc.TunnelMetrics{ - ClientSessionId: stream.SessionID(), + ClientSessionId: string(stream.SessionID()), IngressBytes: ingressBytes.GetValue(), EgressBytes: egressBytes.GetValue(), }) @@ -58,9 +58,10 @@ func (s *state) WatchDial(session *rpc.SessionInfo, server agent.Agent_WatchDial dlog.Debugf(ctx, "WatchDial called from client %s", session.SessionId) defer dlog.Debugf(ctx, "WatchDial ended from client %s", session.SessionId) drCh := make(chan *rpc.DialRequest) - s.dialWatchers.Store(session.SessionId, drCh) + sid := tunnel.SessionID(session.SessionId) + s.dialWatchers.Store(sid, drCh) defer func() { - s.dialWatchers.Delete(session.SessionId) + s.dialWatchers.Delete(sid) }() for { @@ -79,7 +80,7 @@ func (s *state) WatchDial(session *rpc.SessionInfo, server agent.Agent_WatchDial } } -func (s *state) CreateClientStream(ctx context.Context, sessionID string, id tunnel.ConnID, roundTripLatency, dialTimeout time.Duration) (tunnel.Stream, error) { +func (s *state) CreateClientStream(ctx context.Context, sessionID tunnel.SessionID, id tunnel.ConnID, roundTripLatency, dialTimeout time.Duration) (tunnel.Stream, error) { dlog.Debugf(ctx, "Creating tunnel to client %s for id %s", sessionID, id) drCh, ok := s.dialWatchers.Load(sessionID) var stCh <-chan tunnel.Stream diff --git a/cmd/traffic/cmd/agent/state.go b/cmd/traffic/cmd/agent/state.go index 3a7b0ab93f..46f1464f9b 100644 --- a/cmd/traffic/cmd/agent/state.go +++ b/cmd/traffic/cmd/agent/state.go @@ -56,8 +56,8 @@ type state struct { Config ftpPort uint16 sftpPort uint16 - dialWatchers *xsync.MapOf[string, chan *manager.DialRequest] - awaitingForwards *xsync.MapOf[string, *xsync.MapOf[tunnel.ConnID, *awaitingForward]] + dialWatchers *xsync.MapOf[tunnel.SessionID, chan *manager.DialRequest] + awaitingForwards *xsync.MapOf[tunnel.SessionID, *xsync.MapOf[tunnel.ConnID, *awaitingForward]] // The sessionInfo and manager client are needed when forwarders establish their // tunnel to the traffic-manager. @@ -91,8 +91,8 @@ func NewState(config Config) State { return &state{ Config: config, containerStates: make(map[string]ContainerState), - dialWatchers: xsync.NewMapOf[string, chan *manager.DialRequest](), - awaitingForwards: xsync.NewMapOf[string, *xsync.MapOf[tunnel.ConnID, *awaitingForward]](), + dialWatchers: xsync.NewMapOf[tunnel.SessionID, chan *manager.DialRequest](), + awaitingForwards: xsync.NewMapOf[tunnel.SessionID, *xsync.MapOf[tunnel.ConnID, *awaitingForward]](), } } diff --git a/cmd/traffic/cmd/manager/manager.go b/cmd/traffic/cmd/manager/manager.go index a69eb96f97..0b646163af 100644 --- a/cmd/traffic/cmd/manager/manager.go +++ b/cmd/traffic/cmd/manager/manager.go @@ -27,6 +27,7 @@ import ( "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/managerutil" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/mutator" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/namespaces" + "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/state" "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/grpc/server" "github.com/telepresenceio/telepresence/v2/pkg/informer" @@ -247,11 +248,11 @@ func (s *service) servePrometheus(ctx context.Context) error { "Flag to indicate when an intercept is active. 1 for active, 0 for not active.", append(labels, "workload")), ) - s.state.SetAllClientSessionsFinalizer(func(client *rpc.ClientInfo) { + s.state.SetAllClientSessionsFinalizer(func(client *state.ClientSession) { SetGauge(s.state.GetConnectActiveStatus(), client.Name, client.InstallId, nil, 0) }) - s.state.SetAllInterceptsFinalizer(func(client *rpc.ClientInfo, workload *string) { + s.state.SetAllInterceptsFinalizer(func(client *state.ClientSession, workload *string) { SetGauge(s.state.GetInterceptActiveStatus(), client.Name, client.InstallId, workload, 0) }) diff --git a/cmd/traffic/cmd/manager/managerutil/ctx.go b/cmd/traffic/cmd/manager/managerutil/ctx.go index 83cb491fd8..ad3f7753f7 100644 --- a/cmd/traffic/cmd/manager/managerutil/ctx.go +++ b/cmd/traffic/cmd/manager/managerutil/ctx.go @@ -5,27 +5,27 @@ import ( "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/rpc/v2/manager" + "github.com/telepresenceio/telepresence/v2/pkg/tunnel" ) func WithSessionInfo(ctx context.Context, si *manager.SessionInfo) context.Context { if id := si.GetSessionId(); id != "" { - return WithSessionID(ctx, id) + return WithSessionID(ctx, tunnel.SessionID(id)) } return ctx } -func WithSessionID(ctx context.Context, sessionID string) context.Context { +func WithSessionID(ctx context.Context, sessionID tunnel.SessionID) context.Context { ctx = context.WithValue(ctx, sessionContextKey{}, sessionID) ctx = dlog.WithField(ctx, "session_id", sessionID) return ctx } -func GetSessionID(ctx context.Context) string { - id := ctx.Value(sessionContextKey{}) - if id == nil { - return "" +func GetSessionID(ctx context.Context) tunnel.SessionID { + if id, ok := ctx.Value(sessionContextKey{}).(tunnel.SessionID); ok { + return id } - return id.(string) + return "" } type sessionContextKey struct{} diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index df06dcfc99..9317030578 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -161,7 +161,7 @@ func (s *service) GetAgentImageFQN(ctx context.Context, _ *empty.Empty) (*rpc.Ag func (s *service) GetAgentConfig(ctx context.Context, request *rpc.AgentConfigRequest) (*rpc.AgentConfigResponse, error) { dlog.Debug(ctx, "GetAgentConfig called") ctx = managerutil.WithSessionInfo(ctx, request.Session) - sessionID := request.GetSession().GetSessionId() + sessionID := tunnel.SessionID(request.GetSession().GetSessionId()) clientInfo := s.state.GetClient(sessionID) if clientInfo == nil { return nil, status.Errorf(codes.NotFound, "Client session %q not found", sessionID) @@ -214,7 +214,7 @@ func (s *service) ArriveAsClient(ctx context.Context, client *rpc.ClientInfo) (* SetGauge(s.state.GetConnectActiveStatus(), client.Name, client.InstallId, nil, 1) return &rpc.SessionInfo{ - SessionId: s.state.AddClient(client, s.clock.Now()), + SessionId: string(s.state.AddClient(client, s.clock.Now())), ManagerInstallId: s.clusterInfo.ID(), InstallId: &installId, }, nil @@ -237,7 +237,7 @@ func (s *service) ArriveAsAgent(ctx context.Context, agent *rpc.AgentInfo) (*rpc } return &rpc.SessionInfo{ - SessionId: sessionID, + SessionId: string(sessionID), ManagerInstallId: s.clusterInfo.ID(), }, nil } @@ -259,7 +259,7 @@ func (s *service) GetClientConfig(ctx context.Context, _ *empty.Empty) (*rpc.CLI func (s *service) Remain(ctx context.Context, req *rpc.RemainRequest) (*empty.Empty, error) { // ctx = WithSessionInfo(ctx, req.GetSession()) // dlog.Debug(ctx, "Remain called") - sessionID := req.GetSession().GetSessionId() + sessionID := tunnel.SessionID(req.GetSession().GetSessionId()) if ok := s.state.MarkSession(req, s.clock.Now()); !ok { return nil, status.Errorf(codes.NotFound, "Session %q not found", sessionID) } @@ -272,7 +272,7 @@ func (s *service) Remain(ctx context.Context, req *rpc.RemainRequest) (*empty.Em // Depart terminates a session. func (s *service) Depart(ctx context.Context, session *rpc.SessionInfo) (*empty.Empty, error) { ctx = managerutil.WithSessionInfo(ctx, session) - sessionID := session.GetSessionId() + sessionID := tunnel.SessionID(session.GetSessionId()) dlog.Debug(ctx, "Depart called") // There's reason for the caller to wait for this removal to complete. @@ -286,18 +286,18 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa dlog.Debug(ctx, "WatchAgentPods called") defer dlog.Debug(ctx, "WatchAgentPods ended") - clientSession := session.SessionId + clientSession := tunnel.SessionID(session.SessionId) clientInfo := s.state.GetClient(clientSession) if clientInfo == nil { return status.Errorf(codes.NotFound, "Client session %q not found", clientSession) } ns := clientInfo.Namespace - agentsCh := s.state.WatchAgents(ctx, func(_ string, info *rpc.AgentInfo) bool { + agentsCh := s.state.WatchAgents(ctx, func(_ tunnel.SessionID, info *state.AgentSession) bool { return info.Namespace == ns }) interceptsCh := s.state.WatchIntercepts(ctx, func(_ string, info *state.Intercept) bool { - return info.ClientSession.SessionId == clientSession + return info.ClientSession.SessionId == string(clientSession) }) sessionDone, err := s.state.SessionDone(clientSession) if err != nil { @@ -364,19 +364,21 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa func (s *service) WatchAgents(session *rpc.SessionInfo, stream rpc.Manager_WatchAgentsServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), session) dlog.Debug(ctx, "WatchAgents called") - clientInfo := s.state.GetClient(session.SessionId) + clientInfo := s.state.GetClient(tunnel.SessionID(session.SessionId)) if clientInfo == nil { return status.Errorf(codes.NotFound, "Client session %q not found", session.SessionId) } ns := clientInfo.Namespace - return s.watchAgents(ctx, func(_ string, a *rpc.AgentInfo) bool { return a.Namespace == ns }, stream) + return s.watchAgents(ctx, func(_ tunnel.SessionID, a *state.AgentSession) bool { return a.Namespace == ns }, stream) } // WatchAgentsNS notifies a client of the set of known Agents in the namespaces given in the request. func (s *service) WatchAgentsNS(request *rpc.AgentsRequest, stream rpc.Manager_WatchAgentsNSServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), request.Session) dlog.Debug(ctx, "WatchAgentsNS called") - return s.watchAgents(ctx, func(_ string, a *rpc.AgentInfo) bool { return slices.Contains(request.Namespaces, a.Namespace) }, stream) + return s.watchAgents(ctx, func(_ tunnel.SessionID, a *state.AgentSession) bool { + return slices.Contains(request.Namespaces, a.Namespace) + }, stream) } func infosEqual(a, b *rpc.AgentInfo) bool { @@ -409,7 +411,7 @@ func infosEqual(a, b *rpc.AgentInfo) bool { }) } -func (s *service) watchAgents(ctx context.Context, includeAgent func(string, *rpc.AgentInfo) bool, stream rpc.Manager_WatchAgentsServer) error { +func (s *service) watchAgents(ctx context.Context, includeAgent func(tunnel.SessionID, *state.AgentSession) bool, stream rpc.Manager_WatchAgentsServer) error { snapshotCh := s.state.WatchAgents(ctx, includeAgent) sessionDone, err := s.state.SessionDone(managerutil.GetSessionID(ctx)) if err != nil { @@ -431,8 +433,8 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(string, *rp agentSessionIDs := slices.Sorted(maps.Keys(snapshot)) agents := make([]*rpc.AgentInfo, 0, len(agentSessionIDs)) for _, agentSessionID := range agentSessionIDs { - if as := s.state.GetSession(agentSessionID); as != nil && as.Active() { - agents = append(agents, snapshot[agentSessionID]) + if as := s.state.GetAgent(agentSessionID); as != nil { + agents = append(agents, snapshot[agentSessionID].AgentInfo) } } if slices.EqualFunc(agents, lastSnap, infosEqual) { @@ -466,7 +468,7 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(string, *rp // relevant to that client or agent. func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_WatchInterceptsServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), session) - sessionID := session.GetSessionId() + sessionID := tunnel.SessionID(session.GetSessionId()) dlog.Debug(ctx, "WatchIntercepts called") @@ -488,11 +490,10 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W // Don't return intercepts for different agents. return false } - if as := s.state.GetSession(sessionID); as == nil || !as.Active() { - // Session is no longer active + if as := s.state.GetAgent(sessionID); as == nil { + dlog.Debugf(ctx, "WatchIntercepts session no longer active") return false } - // Don't return intercepts that aren't in a "agent-owned" state. switch info.Disposition { case rpc.InterceptDispositionType_WAITING, @@ -510,7 +511,7 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W } else { // sessionID refers to a client session. filter = func(id string, info *state.Intercept) bool { - return info.ClientSession.SessionId == sessionID && + return info.ClientSession.SessionId == string(sessionID) && info.Disposition != rpc.InterceptDispositionType_REMOVED && !state.IsChildIntercept(info.Spec) } @@ -525,10 +526,6 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W dlog.Debugf(ctx, "WatchIntercepts request cancelled") return nil } - if as := s.state.GetSession(sessionID); as == nil || !as.Active() { - dlog.Debugf(ctx, "WatchIntercepts session no longer active") - return nil - } dlog.Tracef(ctx, "WatchIntercepts sending update") intercepts := make([]*rpc.InterceptInfo, 0, len(snapshot)) for _, intercept := range snapshot { @@ -584,7 +581,7 @@ func (s *service) EnsureAgent(ctx context.Context, request *rpc.EnsureAgentReque session := request.GetSession() ctx = managerutil.WithSessionInfo(ctx, session) dlog.Debugf(ctx, "EnsureAgent called") - sessionID := session.GetSessionId() + sessionID := tunnel.SessionID(session.GetSessionId()) client := s.state.GetClient(sessionID) if client == nil { return nil, status.Errorf(codes.NotFound, "Client session %q not found", sessionID) @@ -596,7 +593,11 @@ func (s *service) EnsureAgent(ctx context.Context, request *rpc.EnsureAgentReque if len(as) == 0 { return nil, status.Errorf(codes.Internal, "failed to ensure agent for workload %s: no agents became active", request.Name) } - return &rpc.AgentInfoSnapshot{Agents: as}, nil + rpcAs := make([]*rpc.AgentInfo, len(as)) + for i, a := range as { + rpcAs[i] = a.AgentInfo + } + return &rpc.AgentInfoSnapshot{Agents: rpcAs}, nil } // CreateIntercept lets a client create an intercept. @@ -632,7 +633,7 @@ func (s *service) MakeInterceptID(_ context.Context, sessionID string, name stri if sessionID == "" { return name, nil } else { - if s.state.GetClient(sessionID) == nil { + if s.state.GetClient(tunnel.SessionID(sessionID)) == nil { return "", status.Errorf(codes.NotFound, "Client session %q not found", sessionID) } return sessionID + ":" + name, nil @@ -646,7 +647,7 @@ func (s *service) UpdateIntercept(context.Context, *rpc.UpdateInterceptRequest) // RemoveIntercept lets a client remove an intercept. func (s *service) RemoveIntercept(ctx context.Context, riReq *rpc.RemoveInterceptRequest2) (*empty.Empty, error) { ctx = managerutil.WithSessionInfo(ctx, riReq.GetSession()) - sessionID := riReq.GetSession().GetSessionId() + sessionID := tunnel.SessionID(riReq.GetSession().GetSessionId()) name := riReq.Name dlog.Debugf(ctx, "RemoveIntercept called: %s", name) @@ -658,7 +659,7 @@ func (s *service) RemoveIntercept(ctx context.Context, riReq *rpc.RemoveIntercep SetGauge(s.state.GetInterceptActiveStatus(), client.Name, client.InstallId, &name, 0) - s.state.RemoveIntercept(ctx, sessionID+":"+name) + s.state.RemoveIntercept(ctx, string(sessionID)+":"+name) return &empty.Empty{}, nil } @@ -678,7 +679,7 @@ func (s *service) GetIntercept(ctx context.Context, request *rpc.GetInterceptReq // ReviewIntercept lets an agent approve or reject an intercept. func (s *service) ReviewIntercept(ctx context.Context, rIReq *rpc.ReviewInterceptRequest) (*empty.Empty, error) { ctx = managerutil.WithSessionInfo(ctx, rIReq.GetSession()) - sessionID := rIReq.GetSession().GetSessionId() + sessionID := tunnel.SessionID(rIReq.GetSession().GetSessionId()) ceptID := rIReq.Id agent := s.state.GetAgent(sessionID) @@ -746,7 +747,7 @@ func (s *service) Tunnel(server rpc.Manager_TunnelServer) error { func (s *service) WatchDial(session *rpc.SessionInfo, stream rpc.Manager_WatchDialServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), session) dlog.Debugf(ctx, "WatchDial called") - lrCh := s.state.WatchDial(session.SessionId) + lrCh := s.state.WatchDial(tunnel.SessionID(session.SessionId)) for { select { // connection broken @@ -867,6 +868,7 @@ func (s *service) LookupDNS(ctx context.Context, request *rpc.DNSRequest) (respo } } + sessionID := tunnel.SessionID(request.GetSession().GetSessionId()) tmNamespace := managerutil.GetEnv(ctx).ManagerNamespace noSearchDomain := s.dotClusterDomain var rCode int @@ -882,7 +884,7 @@ func (s *service) LookupDNS(ctx context.Context, request *rpc.DNSRequest) (respo // It's enough to propagate this one to the traffic-manager rCode = state.RcodeNoAgents default: - rrs, rCode, err = s.state.AgentsLookupDNS(ctx, request.GetSession().GetSessionId(), request) + rrs, rCode, err = s.state.AgentsLookupDNS(ctx, sessionID, request) if err != nil { dlog.Errorf(ctx, "AgentsLookupDNS %s %s: %v", request.Name, qtn, err) } else if rCode != state.RcodeNoAgents { @@ -895,7 +897,7 @@ func (s *service) LookupDNS(ctx context.Context, request *rpc.DNSRequest) (respo } if rCode == state.RcodeNoAgents { - client := s.state.GetClient(request.GetSession().GetSessionId()) + client := s.state.GetClient(sessionID) name := request.Name restoreName := false nDots := 0 @@ -951,7 +953,7 @@ func (s *service) AgentLookupDNSResponse(ctx context.Context, response *rpc.DNSA func (s *service) WatchLookupDNS(session *rpc.SessionInfo, stream rpc.Manager_WatchLookupDNSServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), session) dlog.Debugf(ctx, "WatchLookupDNS called") - rqCh := s.state.WatchLookupDNS(session.SessionId) + rqCh := s.state.WatchLookupDNS(tunnel.SessionID(session.SessionId)) for { select { case <-s.ctx.Done(): @@ -1021,7 +1023,7 @@ func (s *service) WatchWorkloads(request *rpc.WorkloadEventsRequest, stream rpc. if request.SessionInfo == nil { return status.Error(codes.InvalidArgument, "SessionInfo is required") } - clientSession := request.SessionInfo.SessionId + clientSession := tunnel.SessionID(request.SessionInfo.SessionId) namespace := request.Namespace if namespace == "" { clientInfo := s.state.GetClient(clientSession) diff --git a/cmd/traffic/cmd/manager/state/consumption.go b/cmd/traffic/cmd/manager/state/consumption.go index 9e52d28fc6..3b7a64c54c 100644 --- a/cmd/traffic/cmd/manager/state/consumption.go +++ b/cmd/traffic/cmd/manager/state/consumption.go @@ -60,27 +60,25 @@ func (m *SessionConsumptionMetrics) SetLastUpdate(t time.Time) { m.lastUpdate.Store(t.UnixNano()) } -func (s *state) GetSessionConsumptionMetrics(sessionID string) *SessionConsumptionMetrics { - if css, ok := s.GetSession(sessionID).(*clientSessionState); ok { - return css.ConsumptionMetrics() +func (s *state) GetSessionConsumptionMetrics(id tunnel.SessionID) *SessionConsumptionMetrics { + if cs := s.GetClient(id); cs != nil { + return cs.ConsumptionMetrics() } return nil } -func (s *state) GetAllSessionConsumptionMetrics() map[string]*SessionConsumptionMetrics { - allSCM := make(map[string]*SessionConsumptionMetrics) - s.sessions.Range(func(sessionID string, sess SessionState) bool { - if css, ok := sess.(*clientSessionState); ok { - allSCM[sessionID] = css.ConsumptionMetrics() - } +func (s *state) GetAllSessionConsumptionMetrics() map[tunnel.SessionID]*SessionConsumptionMetrics { + allSCM := make(map[tunnel.SessionID]*SessionConsumptionMetrics) + s.clients.Range(func(id tunnel.SessionID, cs *ClientSession) bool { + allSCM[id] = cs.ConsumptionMetrics() return true }) return allSCM } func (s *state) AddSessionConsumptionMetrics(metrics *manager.TunnelMetrics) { - cs, ok := s.GetSession(metrics.ClientSessionId).(*clientSessionState) - if ok { + cs := s.GetClient(tunnel.SessionID(metrics.ClientSessionId)) + if cs != nil { cm := cs.consumptionMetrics cm.FromClientBytes.Increment(metrics.IngressBytes) cm.ToClientBytes.Increment(metrics.EgressBytes) @@ -88,10 +86,9 @@ func (s *state) AddSessionConsumptionMetrics(metrics *manager.TunnelMetrics) { } // RefreshSessionConsumptionMetrics refreshes the metrics associated to a specific session. -func (s *state) RefreshSessionConsumptionMetrics(sessionID string) { - css, ok := s.GetSession(sessionID).(*clientSessionState) - if !ok { - return +func (s *state) RefreshSessionConsumptionMetrics(sessionID tunnel.SessionID) { + cs := s.GetClient(sessionID) + if cs != nil { + cs.ConsumptionMetrics().AddTimeSpent() } - css.ConsumptionMetrics().AddTimeSpent() } diff --git a/cmd/traffic/cmd/manager/state/consumption_test.go b/cmd/traffic/cmd/manager/state/consumption_test.go index 67a30137d9..beae91bfaf 100644 --- a/cmd/traffic/cmd/manager/state/consumption_test.go +++ b/cmd/traffic/cmd/manager/state/consumption_test.go @@ -9,11 +9,11 @@ import ( func (s *suiteState) TestRefreshSessionConsumptionMetrics() { // given now := time.Now() - session1 := &clientSessionState{} - session3 := &clientSessionState{} - s.state.sessions.Store("session-1", session1) - s.state.sessions.Store("session-2", &agentSessionState{}) - s.state.sessions.Store("session-3", session3) + session1 := &ClientSession{} + session3 := &ClientSession{} + s.state.clients.Store("session-1", session1) + s.state.agents.Store("session-2", &AgentSession{}) + s.state.clients.Store("session-3", session3) session1.consumptionMetrics = &SessionConsumptionMetrics{} session1.consumptionMetrics.connectDuration.Store(int64(42 * time.Second)) session1.consumptionMetrics.lastUpdate.Store(now.Add(-time.Minute).UnixNano()) @@ -30,10 +30,10 @@ func (s *suiteState) TestRefreshSessionConsumptionMetrics() { s.state.RefreshSessionConsumptionMetrics("session-4") // doesn't exist but shouldn't fail. // then - ccs1, _ := s.state.sessions.Load("session-1") - ccs3, _ := s.state.sessions.Load("session-3") + ccs1, _ := s.state.clients.Load("session-1") + ccs3, _ := s.state.clients.Load("session-3") assert.Len(s.T(), s.state.GetAllSessionConsumptionMetrics(), 2) - assert.True(s.T(), ccs1.(*clientSessionState).ConsumptionMetrics().ConnectDuration() > 42*time.Second) - assert.Equal(s.T(), 41*time.Second, ccs3.(*clientSessionState).ConsumptionMetrics().ConnectDuration()) + assert.True(s.T(), ccs1.ConsumptionMetrics().ConnectDuration() > 42*time.Second) + assert.Equal(s.T(), 41*time.Second, ccs3.ConsumptionMetrics().ConnectDuration()) } diff --git a/cmd/traffic/cmd/manager/state/dns.go b/cmd/traffic/cmd/manager/state/dns.go index 90e4cf5c02..556dcdc708 100644 --- a/cmd/traffic/cmd/manager/state/dns.go +++ b/cmd/traffic/cmd/manager/state/dns.go @@ -11,6 +11,7 @@ import ( rpc "github.com/telepresenceio/telepresence/rpc/v2/manager" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/managerutil" "github.com/telepresenceio/telepresence/v2/pkg/dnsproxy" + "github.com/telepresenceio/telepresence/v2/pkg/tunnel" ) // We can use our own Rcodes in the range that is reserved for private use @@ -20,7 +21,7 @@ const RcodeNoAgents = 3841 // AgentsLookupDNS will send the given request to all agents currently intercepted by the client identified with // the clientSessionID, it will then wait for results to arrive, collect those results, and return the result. -func (s *state) AgentsLookupDNS(ctx context.Context, clientSessionID string, request *rpc.DNSRequest) (dnsproxy.RRs, int, error) { +func (s *state) AgentsLookupDNS(ctx context.Context, clientSessionID tunnel.SessionID, request *rpc.DNSRequest) (dnsproxy.RRs, int, error) { rs := s.agentsLookup(ctx, clientSessionID, request) if len(rs) == 0 { return nil, RcodeNoAgents, nil @@ -45,35 +46,29 @@ func (s *state) AgentsLookupDNS(ctx context.Context, clientSessionID string, req // PostLookupDNSResponse receives lookup responses from an agent and places them in the channel // that corresponds to the lookup request. func (s *state) PostLookupDNSResponse(ctx context.Context, response *rpc.DNSAgentResponse) { - request := response.GetRequest() - rid := requestId(request) - s.mu.RLock() - as, ok := s.GetSession(response.GetSession().SessionId).(*agentSessionState) - if ok { - var rch chan<- *rpc.DNSResponse - if rch, ok = as.dnsResponses[rid]; ok { + rid := requestId(response.GetRequest()) + as := s.GetAgent(tunnel.SessionID(response.GetSession().SessionId)) + if as != nil { + if rch, ok := as.dnsResponses.Load(rid); ok { select { case rch <- response.GetResponse(): default: - ok = false } } - } - s.mu.RUnlock() - if !ok { + } else { dlog.Debugf(ctx, "attempted to post lookup response failed because there was no recipient. ID=%s", rid) } } -func (s *state) WatchLookupDNS(agentSessionID string) <-chan *rpc.DNSRequest { - ss, ok := s.GetSession(agentSessionID).(*agentSessionState) - if ok { - return ss.dnsRequests +func (s *state) WatchLookupDNS(agentSessionID tunnel.SessionID) <-chan *rpc.DNSRequest { + as := s.GetAgent(agentSessionID) + if as != nil { + return as.dnsRequests } return nil } -func (s *state) agentsLookup(ctx context.Context, clientSessionID string, request *rpc.DNSRequest) []*rpc.DNSResponse { +func (s *state) agentsLookup(ctx context.Context, clientSessionID tunnel.SessionID, request *rpc.DNSRequest) []*rpc.DNSResponse { agents := s.getAgentsInterceptedByClient(clientSessionID) if len(agents) == 0 { if client, ok := s.clients.Load(clientSessionID); ok { @@ -101,7 +96,7 @@ func (s *state) agentsLookup(ctx context.Context, clientSessionID string, reques wg := sync.WaitGroup{} wg.Add(aCount) for aID := range agents { - go func(aID string) { + go func(aID tunnel.SessionID) { rid := requestId(request) defer func() { s.endLookup(aID, rid) @@ -134,48 +129,37 @@ func (s *state) agentsLookup(ctx context.Context, clientSessionID string, reques return rs } -func (s *state) startLookup(agentSessionID, rid string, request *rpc.DNSRequest) <-chan *rpc.DNSResponse { - var ( - rch chan *rpc.DNSResponse - as *agentSessionState - ok bool - ) - s.mu.Lock() - if as, ok = s.GetSession(agentSessionID).(*agentSessionState); ok { - if rch, ok = as.dnsResponses[rid]; !ok { - rch = make(chan *rpc.DNSResponse) - as.dnsResponses[rid] = rch - } - } - s.mu.Unlock() - if as != nil { - // the as.dnsRequests channel may be closed at this point, so guard for panic - func() { - defer func() { - if r := recover(); r != nil { - select { - case <-rch: - // rch is already closed - default: - close(rch) - } - } - }() - as.dnsRequests <- request - }() +func (s *state) startLookup(agentSessionID tunnel.SessionID, rid string, request *rpc.DNSRequest) <-chan *rpc.DNSResponse { + as := s.GetAgent(agentSessionID) + if as == nil { + return nil } + rch, _ := as.dnsResponses.LoadOrCompute(rid, func() chan *rpc.DNSResponse { + return make(chan *rpc.DNSResponse) + }) + + // The as.dnsRequests channel may be closed at this point, so guard for panic. And no, we can't read that + // channel to check if it is closed, because we're not the intended recipient. + defer func() { + if r := recover(); r != nil { + select { + case <-rch: + // rch is already closed + default: + close(rch) + } + } + }() + as.dnsRequests <- request return rch } -func (s *state) endLookup(agentSessionID, rid string) { - s.mu.Lock() - if as, ok := s.GetSession(agentSessionID).(*agentSessionState); ok { - if rch, ok := as.dnsResponses[rid]; ok { - delete(as.dnsResponses, rid) +func (s *state) endLookup(agentSessionID tunnel.SessionID, rid string) { + if as := s.GetAgent(agentSessionID); as != nil { + if rch, loaded := as.dnsResponses.LoadAndDelete(rid); loaded { close(rch) } } - s.mu.Unlock() } func requestId(request *rpc.DNSRequest) string { diff --git a/cmd/traffic/cmd/manager/state/intercept.go b/cmd/traffic/cmd/manager/state/intercept.go index 5bab3acf81..8ff1bad087 100644 --- a/cmd/traffic/cmd/manager/state/intercept.go +++ b/cmd/traffic/cmd/manager/state/intercept.go @@ -31,6 +31,7 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/errcat" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" + "github.com/telepresenceio/telepresence/v2/pkg/tunnel" ) // PrepareIntercept ensures that the given request can be matched against the intercept configuration of @@ -243,9 +244,9 @@ func (s *state) preparePorts(ac *agentconfig.Sidecar, cn *agentconfig.Container, return nil } -func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptRequest) (*rpc.ClientInfo, *rpc.InterceptInfo, error) { +func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptRequest) (*ClientSession, *rpc.InterceptInfo, error) { clientSession := cir.Session - sessionID := clientSession.SessionId + sessionID := tunnel.SessionID(clientSession.SessionId) client := s.GetClient(sessionID) if client == nil { return nil, nil, status.Errorf(codes.NotFound, "session %q not found", sessionID) @@ -375,14 +376,14 @@ func (s *state) AddInterceptFinalizer(interceptID string, finalizer InterceptFin // getAgentsInterceptedByClient returns the session IDs for each agent that are currently // intercepted by the client with the given client session ID. -func (s *state) getAgentsInterceptedByClient(clientSessionID string) map[string]*rpc.AgentInfo { +func (s *state) getAgentsInterceptedByClient(clientSessionID tunnel.SessionID) map[tunnel.SessionID]*AgentSession { intercepts := s.intercepts.LoadMatching(func(_ string, ii *Intercept) bool { - return ii.ClientSession.SessionId == clientSessionID + return ii.ClientSession.SessionId == string(clientSessionID) }) if len(intercepts) == 0 { return nil } - return s.LoadMatchingAgents(func(_ string, ai *rpc.AgentInfo) bool { + return s.LoadMatchingAgents(func(_ tunnel.SessionID, ai *AgentSession) bool { for _, ii := range intercepts { if ai.Name == ii.Spec.Agent && ai.Namespace == ii.Spec.Namespace { return true @@ -392,7 +393,7 @@ func (s *state) getAgentsInterceptedByClient(clientSessionID string) map[string] }) } -func (s *state) EnsureAgent(ctx context.Context, n, ns string) (as []*rpc.AgentInfo, err error) { +func (s *state) EnsureAgent(ctx context.Context, n, ns string) (as []*AgentSession, err error) { var wl k8sapi.Workload wl, err = agentmap.GetWorkload(ctx, n, ns, "") if err != nil { @@ -410,14 +411,14 @@ func (s *state) ValidateCreateAgent(context.Context, k8sapi.Workload, agentconfi } // sortAgents will sort the given AgentInfo based on pod name. -func sortAgents(as []*rpc.AgentInfo) { +func sortAgents(as []*AgentSession) { sort.Slice(as, func(i, j int) bool { return as[i].PodName < as[j].PodName }) } func (s *state) ensureAgent(parentCtx context.Context, wl k8sapi.Workload, extended, dryRun bool, spec *rpc.InterceptSpec, rp agentconfig.ReplacePolicy) ( - ac *agentconfig.Sidecar, as []*rpc.AgentInfo, err error, + ac *agentconfig.Sidecar, as []*AgentSession, err error, ) { if agentmap.TrafficManagerSelector.Matches(labels.Set(wl.GetLabels())) { msg := fmt.Sprintf("deployment %s.%s is the Telepresence Traffic Manager. It can not have a traffic-agent", wl.GetName(), wl.GetNamespace()) @@ -436,10 +437,10 @@ func (s *state) ensureAgent(parentCtx context.Context, wl k8sapi.Workload, exten return nil, nil, err } ac = sce.AgentConfig() - am := s.LoadMatchingAgents(func(_ string, ai *rpc.AgentInfo) bool { + am := s.LoadMatchingAgents(func(_ tunnel.SessionID, ai *AgentSession) bool { return ai.Name == ac.AgentName && ai.Namespace == ac.Namespace }) - as = make([]*rpc.AgentInfo, len(am)) + as = make([]*AgentSession, len(am)) i := 0 for _, found := range am { as[i] = found @@ -716,11 +717,11 @@ func watchFailedInjectionEvents(ctx context.Context, name, namespace string) (<- return ec, nil } -func (s *state) waitForAgents(ctx context.Context, ac *agentconfig.Sidecar, failedCreateCh <-chan *events.Event) ([]*rpc.AgentInfo, error) { +func (s *state) waitForAgents(ctx context.Context, ac *agentconfig.Sidecar, failedCreateCh <-chan *events.Event) ([]*AgentSession, error) { name := ac.AgentName namespace := ac.Namespace dlog.Debugf(ctx, "Waiting for agent %s.%s", name, namespace) - snapshotCh := s.WatchAgents(ctx, func(sessionID string, agent *rpc.AgentInfo) bool { + snapshotCh := s.WatchAgents(ctx, func(_ tunnel.SessionID, agent *AgentSession) bool { return agent.Name == name && agent.Namespace == namespace }) failedContainerRx := regexp.MustCompile(`restarting failed container (\S+) in pod ([0-9A-Za-z_-]+)_` + namespace) @@ -787,7 +788,7 @@ func (s *state) waitForAgents(ctx context.Context, ac *agentconfig.Sidecar, fail if len(snapshot) == 0 { continue } - as := make([]*rpc.AgentInfo, 0, len(snapshot)) + as := make([]*AgentSession, 0, len(snapshot)) for _, a := range snapshot { if mm.IsInactive(types.UID(a.PodUid)) { dlog.Debugf(ctx, "Agent %s(%s) is blacklisted", a.PodName, a.PodIp) diff --git a/cmd/traffic/cmd/manager/state/presence_test.go b/cmd/traffic/cmd/manager/state/presence_test.go index 777b1ad0af..64ebfe4efa 100644 --- a/cmd/traffic/cmd/manager/state/presence_test.go +++ b/cmd/traffic/cmd/manager/state/presence_test.go @@ -7,6 +7,7 @@ import ( "github.com/datawire/dlib/dlog" rpc "github.com/telepresenceio/telepresence/rpc/v2/manager" + "github.com/telepresenceio/telepresence/v2/pkg/tunnel" ) func TestPresence(t *testing.T) { @@ -21,7 +22,7 @@ func TestPresence(t *testing.T) { // A@0 B@0 - isPresent := func(sessionID string) bool { + isPresent := func(sessionID tunnel.SessionID) bool { _, err := p.SessionDone(sessionID) return err == nil } @@ -36,12 +37,12 @@ func TestPresence(t *testing.T) { a.Equal("item-a", p.GetClient(sa).Name) a.Nil(p.GetClient("c")) - a.True(p.MarkSession(&rpc.RemainRequest{Session: &rpc.SessionInfo{SessionId: sa}}, now)) - a.True(p.MarkSession(&rpc.RemainRequest{Session: &rpc.SessionInfo{SessionId: sb}}, now)) + a.True(p.MarkSession(&rpc.RemainRequest{Session: &rpc.SessionInfo{SessionId: string(sa)}}, now)) + a.True(p.MarkSession(&rpc.RemainRequest{Session: &rpc.SessionInfo{SessionId: string(sb)}}, now)) a.False(p.MarkSession(&rpc.RemainRequest{Session: &rpc.SessionInfo{SessionId: "c"}}, now)) now = now.Add(time.Second) - a.True(p.MarkSession(&rpc.RemainRequest{Session: &rpc.SessionInfo{SessionId: sb}}, now)) + a.True(p.MarkSession(&rpc.RemainRequest{Session: &rpc.SessionInfo{SessionId: string(sb)}}, now)) sc := p.AddClient(&rpc.ClientInfo{Name: "item-c"}, now) // A@0 B@1 C@1 @@ -52,7 +53,7 @@ func TestPresence(t *testing.T) { a.False(isPresent("d")) collected := make([]string, 0, 3) - p.EachClient(func(id string, item *rpc.ClientInfo) bool { + p.EachClient(func(id tunnel.SessionID, item *ClientSession) bool { collected = append(collected, fmt.Sprintf("%s/%v", id, item.Name)) return true }) diff --git a/cmd/traffic/cmd/manager/state/session.go b/cmd/traffic/cmd/manager/state/session.go index 28f8df3915..57b6535f03 100644 --- a/cmd/traffic/cmd/manager/state/session.go +++ b/cmd/traffic/cmd/manager/state/session.go @@ -17,6 +17,7 @@ import ( const AgentSessionIDPrefix = "agent:" type SessionState interface { + ID() tunnel.SessionID Active() bool Cancel() AwaitingBidiMapOwnerSessionID(stream tunnel.Stream) string @@ -35,6 +36,7 @@ type awaitingBidiPipe struct { } type sessionState struct { + id tunnel.SessionID doneCh <-chan struct{} cancel context.CancelFunc lastMarked int64 @@ -42,14 +44,18 @@ type sessionState struct { dials chan *rpc.DialRequest } +func (s *sessionState) ID() tunnel.SessionID { + return s.id +} + // EstablishBidiPipe registers the given stream as waiting for a matching stream to arrive in a call // to Tunnel, sends a DialRequest to the owner of this sessionState, and then waits. When the call // arrives, a BidiPipe connecting the two streams is returned. -func (ss *sessionState) EstablishBidiPipe(ctx context.Context, stream tunnel.Stream) (tunnel.Endpoint, error) { +func (s *sessionState) EstablishBidiPipe(ctx context.Context, stream tunnel.Stream) (tunnel.Endpoint, error) { // Dispatch directly to agent and let the dial happen there bidiPipeCh := make(chan tunnel.Endpoint) id := stream.ID() - ss.awaitingBidiPipeMap.Store(id, awaitingBidiPipe{ctx: ctx, stream: stream, bidiPipeCh: bidiPipeCh}) + s.awaitingBidiPipeMap.Store(id, awaitingBidiPipe{ctx: ctx, stream: stream, bidiPipeCh: bidiPipeCh}) // Send dial request to the client/agent dr := &rpc.DialRequest{ @@ -58,9 +64,9 @@ func (ss *sessionState) EstablishBidiPipe(ctx context.Context, stream tunnel.Str DialTimeout: int64(stream.DialTimeout()), } select { - case <-ss.Done(): + case <-s.Done(): return nil, status.Error(codes.Canceled, "session cancelled") - case ss.dials <- dr: + case s.dials <- dr: } // Wait for the client/agent to connect. Allow extra time for the call @@ -69,15 +75,15 @@ func (ss *sessionState) EstablishBidiPipe(ctx context.Context, stream tunnel.Str select { case <-ctx.Done(): return nil, status.Error(codes.DeadlineExceeded, "timeout while establishing bidipipe") - case <-ss.Done(): + case <-s.Done(): return nil, status.Error(codes.Canceled, "session cancelled") case bidi := <-bidiPipeCh: return bidi, nil } } -func (ss *sessionState) AwaitingBidiMapOwnerSessionID(stream tunnel.Stream) string { - if abp, ok := ss.awaitingBidiPipeMap.Load(stream.ID()); ok { +func (s *sessionState) AwaitingBidiMapOwnerSessionID(stream tunnel.Stream) tunnel.SessionID { + if abp, ok := s.awaitingBidiPipeMap.Load(stream.ID()); ok { return abp.stream.SessionID() } return "" @@ -86,7 +92,7 @@ func (ss *sessionState) AwaitingBidiMapOwnerSessionID(stream tunnel.Stream) stri // OnConnect checks if a stream is waiting for the given stream to arrive in order to create a BidiPipe. // If that's the case, the BidiPipe is created, started, and returned by both this method and the EstablishBidiPipe // method that registered the waiting stream. Otherwise, this method returns nil. -func (ss *sessionState) OnConnect( +func (s *sessionState) OnConnect( ctx context.Context, stream tunnel.Stream, counter *int32, @@ -94,7 +100,7 @@ func (ss *sessionState) OnConnect( ) (tunnel.Endpoint, error) { id := stream.ID() // abp is a session corresponding to an end user machine - abp, ok := ss.awaitingBidiPipeMap.LoadAndDelete(id) + abp, ok := s.awaitingBidiPipeMap.LoadAndDelete(id) if !ok { return nil, nil } @@ -110,41 +116,38 @@ func (ss *sessionState) OnConnect( defer close(abp.bidiPipeCh) select { - case <-ss.Done(): + case <-s.Done(): return nil, status.Error(codes.Canceled, "session cancelled") case abp.bidiPipeCh <- bidiPipe: return bidiPipe, nil } } -func (ss *sessionState) Active() bool { - return true -} - -func (ss *sessionState) Cancel() { - ss.cancel() - close(ss.dials) +func (s *sessionState) Cancel() { + s.cancel() + close(s.dials) } -func (ss *sessionState) Dials() <-chan *rpc.DialRequest { - return ss.dials +func (s *sessionState) Dials() <-chan *rpc.DialRequest { + return s.dials } -func (ss *sessionState) Done() <-chan struct{} { - return ss.doneCh +func (s *sessionState) Done() <-chan struct{} { + return s.doneCh } -func (ss *sessionState) LastMarked() time.Time { - return time.Unix(0, atomic.LoadInt64(&ss.lastMarked)) +func (s *sessionState) LastMarked() time.Time { + return time.Unix(0, atomic.LoadInt64(&s.lastMarked)) } -func (ss *sessionState) SetLastMarked(lastMarked time.Time) { - atomic.StoreInt64(&ss.lastMarked, lastMarked.UnixNano()) +func (s *sessionState) SetLastMarked(lastMarked time.Time) { + atomic.StoreInt64(&s.lastMarked, lastMarked.UnixNano()) } -func newSessionState(ctx context.Context, now time.Time) sessionState { +func newSessionState(ctx context.Context, id tunnel.SessionID, now time.Time) sessionState { ctx, cancel := context.WithCancel(ctx) return sessionState{ + id: id, doneCh: ctx.Done(), cancel: cancel, lastMarked: now.UnixNano(), @@ -153,46 +156,50 @@ func newSessionState(ctx context.Context, now time.Time) sessionState { } } -type clientSessionState struct { +type ClientSession struct { + *rpc.ClientInfo sessionState pool *tunnel.Pool consumptionMetrics *SessionConsumptionMetrics } -func (css *clientSessionState) ConsumptionMetrics() *SessionConsumptionMetrics { - return css.consumptionMetrics +func (cs *ClientSession) ConsumptionMetrics() *SessionConsumptionMetrics { + return cs.consumptionMetrics } -func newClientSessionState(ctx context.Context, ts time.Time) *clientSessionState { - return &clientSessionState{ - sessionState: newSessionState(ctx, ts), - pool: tunnel.NewPool(), - +func newClientSessionState(ctx context.Context, id tunnel.SessionID, ci *rpc.ClientInfo, ts time.Time) *ClientSession { + return &ClientSession{ + ClientInfo: ci, + sessionState: newSessionState(ctx, id, ts), + pool: tunnel.NewPool(), consumptionMetrics: NewSessionConsumptionMetrics(), } } -type agentSessionState struct { +type AgentSession struct { + *rpc.AgentInfo sessionState dnsRequests chan *rpc.DNSRequest - dnsResponses map[string]chan *rpc.DNSResponse + dnsResponses *xsync.MapOf[string, chan *rpc.DNSResponse] } -func newAgentSessionState(ctx context.Context, ts time.Time) *agentSessionState { - as := &agentSessionState{ - sessionState: newSessionState(ctx, ts), +func newAgentSessionState(ctx context.Context, id tunnel.SessionID, ai *rpc.AgentInfo, ts time.Time) *AgentSession { + as := &AgentSession{ + AgentInfo: ai, + sessionState: newSessionState(ctx, id, ts), dnsRequests: make(chan *rpc.DNSRequest), - dnsResponses: make(map[string]chan *rpc.DNSResponse), + dnsResponses: xsync.NewMapOf[string, chan *rpc.DNSResponse](), } return as } -func (ss *agentSessionState) Cancel() { - close(ss.dnsRequests) - for k, lr := range ss.dnsResponses { - delete(ss.dnsResponses, k) +func (as *AgentSession) Cancel() { + close(as.dnsRequests) + as.dnsResponses.Range(func(k string, lr chan *rpc.DNSResponse) bool { + as.dnsResponses.Delete(k) close(lr) - } - ss.sessionState.Cancel() + return true + }) + as.sessionState.Cancel() } diff --git a/cmd/traffic/cmd/manager/state/state.go b/cmd/traffic/cmd/manager/state/state.go index 0d4a290f54..9109a9a4ba 100644 --- a/cmd/traffic/cmd/manager/state/state.go +++ b/cmd/traffic/cmd/manager/state/state.go @@ -7,7 +7,6 @@ import ( "os" "slices" "strings" - "sync" "sync/atomic" "time" @@ -63,12 +62,12 @@ func (is *Intercept) terminate(ctx context.Context) { } type State interface { - AddAgent(context.Context, *rpc.AgentInfo, time.Time) (string, error) - AddClient(*rpc.ClientInfo, time.Time) string - AddIntercept(context.Context, *rpc.CreateInterceptRequest) (*rpc.ClientInfo, *rpc.InterceptInfo, error) + AddAgent(context.Context, *rpc.AgentInfo, time.Time) (tunnel.SessionID, error) + AddClient(*rpc.ClientInfo, time.Time) tunnel.SessionID + AddIntercept(context.Context, *rpc.CreateInterceptRequest) (*ClientSession, *rpc.InterceptInfo, error) AddInterceptFinalizer(string, InterceptFinalizer) error AddSessionConsumptionMetrics(metrics *rpc.TunnelMetrics) - AgentsLookupDNS(context.Context, string, *rpc.DNSRequest) (dnsproxy.RRs, int, error) + AgentsLookupDNS(context.Context, tunnel.SessionID, *rpc.DNSRequest) (dnsproxy.RRs, int, error) CountAgents() int CountClients() int CountIntercepts() int @@ -77,13 +76,12 @@ type State interface { CountTunnelIngress() uint64 CountTunnelEgress() uint64 ExpireSessions(context.Context, time.Time, time.Time) - GetAgent(sessionID string) *rpc.AgentInfo + GetAgent(sessionID tunnel.SessionID) *AgentSession GetOrGenerateAgentConfig(ctx context.Context, name, namespace string) (agentconfig.SidecarExt, error) - EachClient(f func(string, *rpc.ClientInfo) bool) - GetClient(sessionID string) *rpc.ClientInfo - GetSession(string) SessionState - GetSessionConsumptionMetrics(string) *SessionConsumptionMetrics - GetAllSessionConsumptionMetrics() map[string]*SessionConsumptionMetrics + EachClient(f func(tunnel.SessionID, *ClientSession) bool) + GetClient(sessionID tunnel.SessionID) *ClientSession + GetSessionConsumptionMetrics(tunnel.SessionID) *SessionConsumptionMetrics + GetAllSessionConsumptionMetrics() map[tunnel.SessionID]*SessionConsumptionMetrics GetIntercept(string) (*Intercept, bool) GetConnectCounter() *prometheus.CounterVec GetConnectActiveStatus() *prometheus.GaugeVec @@ -93,11 +91,11 @@ type State interface { MarkSession(*rpc.RemainRequest, time.Time) bool NewInterceptInfo(string, *rpc.CreateInterceptRequest) *Intercept PostLookupDNSResponse(context.Context, *rpc.DNSAgentResponse) - EnsureAgent(context.Context, string, string) ([]*rpc.AgentInfo, error) + EnsureAgent(context.Context, string, string) ([]*AgentSession, error) PrepareIntercept(context.Context, *rpc.CreateInterceptRequest) (*rpc.PreparedIntercept, error) RemoveIntercept(context.Context, string) - RemoveSession(context.Context, string) - SessionDone(string) (<-chan struct{}, error) + RemoveSession(context.Context, tunnel.SessionID) + SessionDone(tunnel.SessionID) (<-chan struct{}, error) SetTempLogLevel(context.Context, *rpc.LogLevelRequest) SetAllClientSessionsFinalizer(finalizer allClientSessionsFinalizer) SetAllInterceptsFinalizer(finalizer allInterceptsFinalizer) @@ -107,23 +105,23 @@ type State interface { interceptStatusGaugeVec *prometheus.GaugeVec) Tunnel(context.Context, tunnel.Stream) error UpdateIntercept(string, func(*Intercept)) *Intercept - RefreshSessionConsumptionMetrics(sessionID string) + RefreshSessionConsumptionMetrics(sessionID tunnel.SessionID) ValidateAgentImage(string, bool) error WaitForTempLogLevel(rpc.Manager_WatchLogLevelServer) error - WatchAgents(context.Context, func(sessionID string, agent *rpc.AgentInfo) bool) <-chan map[string]*rpc.AgentInfo - WatchDial(sessionID string) <-chan *rpc.DialRequest + WatchAgents(context.Context, func(tunnel.SessionID, *AgentSession) bool) <-chan map[tunnel.SessionID]*AgentSession + WatchDial(tunnel.SessionID) <-chan *rpc.DialRequest WatchIntercepts(context.Context, func(sessionID string, intercept *Intercept) bool) <-chan map[string]*Intercept WatchWorkloads(ctx context.Context, namespace string) (ch <-chan []workload.Event, err error) - WatchLookupDNS(string) <-chan *rpc.DNSRequest + WatchLookupDNS(id tunnel.SessionID) <-chan *rpc.DNSRequest ValidateCreateAgent(context.Context, k8sapi.Workload, agentconfig.SidecarExt) error - NewWorkloadInfoWatcher(clientSession, namespace string) WorkloadInfoWatcher + NewWorkloadInfoWatcher(clientSession tunnel.SessionID, namespace string) WorkloadInfoWatcher ManagesNamespace(context.Context, string) bool UninstallAgents(context.Context, *rpc.UninstallAgentsRequest) error } type ( - allClientSessionsFinalizer func(client *rpc.ClientInfo) - allInterceptsFinalizer func(client *rpc.ClientInfo, workload *string) + allClientSessionsFinalizer func(client *ClientSession) + allInterceptsFinalizer func(client *ClientSession, workload *string) ) // state is the total state of the Traffic Manager. A zero state is invalid; you must call @@ -135,24 +133,9 @@ type state struct { allClientSessionsFinalizer allClientSessionsFinalizer allInterceptsFinalizer allInterceptsFinalizer - - mu sync.RWMutex - // Things protected by 'mu': While the watchable.WhateverMaps have their own locking to - // protect against memory corruption and ensure serialization for watches, we need to do our - // own locking here to ensure consistency between the various maps: - // - // 3. `port` needs to be updated in-sync with `intercepts` - // 5. `intercepts` needs to be pruned in-sync with `clients` (based on - // `intercept.ClientSession.SessionId`) - // 6. `intercepts` needs to be pruned in-sync with `agents` (based on - // `agent.Name == intercept.Spec.Agent`) - // 7. `cfgMapLocks` access must be concurrency protected - // 8. `cachedAgentImage` access must be concurrency protected - // 9. `interceptState` must be concurrency protected and updated/deleted in sync with intercepts - intercepts *watchable.Map[string, *Intercept] // info for intercepts, keyed by intercept id - agents *watchable.Map[string, *rpc.AgentInfo] // info for agent sessions, keyed by session id - clients *xsync.MapOf[string, *rpc.ClientInfo] // info for client sessions, keyed by session id - sessions *xsync.MapOf[string, SessionState] // info for all sessions, keyed by session id + intercepts *watchable.Map[string, *Intercept] // info for intercepts, keyed by intercept id + agents *watchable.Map[tunnel.SessionID, *AgentSession] // info for agent sessions, keyed by session id + clients *xsync.MapOf[tunnel.SessionID, *ClientSession] // info for client sessions, keyed by session id timedLogLevel log.TimedLevel llSubs *loglevelSubscribers workloadWatchers *xsync.MapOf[string, workload.Watcher] // workload watchers, created on demand and keyed by namespace @@ -178,8 +161,8 @@ func interceptEqual(a, b *Intercept) bool { return proto.Equal(a.InterceptInfo, b.InterceptInfo) } -func agentsEqual(a, b *rpc.AgentInfo) bool { - return proto.Equal(a, b) +func agentsEqual(a, b *AgentSession) bool { + return proto.Equal(a.AgentInfo, b.AgentInfo) } func NewState(ctx context.Context) State { @@ -187,9 +170,8 @@ func NewState(ctx context.Context) State { s := &state{ backgroundCtx: ctx, intercepts: watchable.NewMap[string, *Intercept](interceptEqual, time.Millisecond), - agents: watchable.NewMap[string, *rpc.AgentInfo](agentsEqual, time.Millisecond), - clients: xsync.NewMapOf[string, *rpc.ClientInfo](), - sessions: xsync.NewMapOf[string, SessionState](), + agents: watchable.NewMap[tunnel.SessionID, *AgentSession](agentsEqual, time.Millisecond), + clients: xsync.NewMapOf[tunnel.SessionID, *ClientSession](), workloadWatchers: xsync.NewMapOf[string, workload.Watcher](), timedLogLevel: log.NewTimedLevel(loglevel, log.SetLevel), llSubs: newLoglevelSubscribers(), @@ -216,15 +198,25 @@ func NewState(ctx context.Context) State { // pruneSessions will remove all sessions that belong to namespaces that are no longer managed. func (s *state) pruneSessions(ctx context.Context) { nss := namespaces.Get(ctx) - var sids []string - s.clients.Range(func(s string, c *rpc.ClientInfo) bool { + var sids []tunnel.SessionID + s.clients.Range(func(id tunnel.SessionID, c *ClientSession) bool { + if !slices.Contains(nss, c.Namespace) { + sids = append(sids, id) + } + return true + }) + for _, sid := range sids { + s.removeClientSession(ctx, sid) + } + clear(sids) + s.agents.Range(func(s tunnel.SessionID, c *AgentSession) bool { if !slices.Contains(nss, c.Namespace) { sids = append(sids, s) } return true }) for _, sid := range sids { - s.RemoveSession(ctx, sid) + s.removeAgentSession(ctx, sid) } } @@ -272,9 +264,9 @@ func (s *state) checkAgentsForIntercept(intercept *Intercept) (errCode rpc.Inter var agentList []*rpc.AgentInfo agentName := intercept.Spec.Agent ns := intercept.Spec.Namespace - s.EachAgent(func(_ string, ai *rpc.AgentInfo) bool { + s.EachAgent(func(_ tunnel.SessionID, ai *AgentSession) bool { if ai.Name == agentName && ai.Namespace == ns { - agentList = append(agentList, ai) + agentList = append(agentList, ai.AgentInfo) } return true }) @@ -301,42 +293,61 @@ func (s *state) checkAgentsForIntercept(intercept *Intercept) (errCode rpc.Inter // MarkSession marks a session as being present at the indicated time. Returns true if everything goes OK, // returns false if the given session ID does not exist. func (s *state) MarkSession(req *rpc.RemainRequest, now time.Time) (ok bool) { - if sess := s.GetSession(req.Session.SessionId); sess != nil { - sess.SetLastMarked(now) + id := tunnel.SessionID(req.Session.SessionId) + if cs, ok := s.clients.Load(id); ok { + cs.SetLastMarked(now) + return true + } else if as, ok := s.agents.Load(id); ok { + as.SetLastMarked(now) return true } return false } -func (s *state) GetSession(sessionID string) SessionState { - sess, _ := s.sessions.Load(sessionID) - return sess +// RemoveSession removes an AgentSession from the set of present session IDs. +func (s *state) RemoveSession(ctx context.Context, id tunnel.SessionID) { + if _, ok := s.clients.Load(id); ok { + s.removeClientSession(ctx, id) + } else { + s.removeAgentSession(ctx, id) + } } -// RemoveSession removes a session from the set of present session IDs. -func (s *state) RemoveSession(ctx context.Context, sessionID string) { - s.sessions.Compute(sessionID, func(sess SessionState, ok bool) (SessionState, bool) { - if !ok { +// removeAgentSession removes an AgentSession from the set of present session IDs. +func (s *state) removeAgentSession(ctx context.Context, id tunnel.SessionID) { + s.agents.Compute(id, func(agent *AgentSession, loaded bool) (*AgentSession, bool) { + if !loaded { return nil, true } - dlog.Debugf(ctx, "Session %s removed. Explicit removal", sessionID) + dlog.Debugf(ctx, "AgentSession %s removed. Explicit removal", id) // kill the session - defer sess.Cancel() - if agent, isAgent := s.agents.LoadAndDelete(sessionID); isAgent { - s.consolidateAgentSessionIntercepts(ctx, agent) - } else if client, isClient := s.clients.LoadAndDelete(sessionID); isClient { - s.gcClientSessionIntercepts(ctx, sessionID, client) - scm := sess.(*clientSessionState).consumptionMetrics - atomic.AddUint64(&s.tunnelIngressCounter, scm.FromClientBytes.GetValue()) - atomic.AddUint64(&s.tunnelEgressCounter, scm.ToClientBytes.GetValue()) - s.allClientSessionsFinalizerCall(client) + defer agent.Cancel() + s.consolidateAgentSessionIntercepts(ctx, agent) + return nil, true + }) +} + +// removeClientSession removes an AgentSession from the set of present session IDs. +func (s *state) removeClientSession(ctx context.Context, id tunnel.SessionID) { + s.clients.Compute(id, func(client *ClientSession, loaded bool) (*ClientSession, bool) { + if !loaded { + return nil, true } + dlog.Debugf(ctx, "ClientSession %s removed. Explicit removal", id) + + // kill the session + defer client.Cancel() + s.gcClientSessionIntercepts(ctx, id, client) + scm := client.consumptionMetrics + atomic.AddUint64(&s.tunnelIngressCounter, scm.FromClientBytes.GetValue()) + atomic.AddUint64(&s.tunnelEgressCounter, scm.ToClientBytes.GetValue()) + s.allClientSessionsFinalizerCall(client) return nil, true }) } -func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *rpc.AgentInfo) { +func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *AgentSession) { dlog.Debugf(ctx, "Consolidating intercepts after removal of agent %s(%s)", agent.PodName, agent.PodIp) mutator.GetMap(s.backgroundCtx).Inactivate(types.UID(agent.PodUid)) s.intercepts.Range(func(interceptID string, intercept *Intercept) bool { @@ -366,13 +377,13 @@ func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *rp }) } -func (s *state) gcClientSessionIntercepts(ctx context.Context, sessionID string, client *rpc.ClientInfo) { +func (s *state) gcClientSessionIntercepts(ctx context.Context, id tunnel.SessionID, client *ClientSession) { // GC all intercepts for the client session (intercept.ClientSession.SessionId) s.intercepts.Range(func(interceptID string, intercept *Intercept) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED { return true } - if intercept.ClientSession.SessionId == sessionID { + if tunnel.SessionID(intercept.ClientSession.SessionId) == id { // Client went away: // Delete it. wl := strings.SplitN(interceptID, ":", 2)[1] @@ -386,13 +397,17 @@ func (s *state) gcClientSessionIntercepts(ctx context.Context, sessionID string, // ExpireSessions prunes any sessions that haven't had a MarkSession heartbeat since // respective given 'moment'. func (s *state) ExpireSessions(ctx context.Context, clientMoment, agentMoment time.Time) { - s.sessions.Range(func(id string, sess SessionState) bool { - moment := agentMoment - if _, ok := sess.(*clientSessionState); ok { - moment = clientMoment + s.clients.Range(func(id tunnel.SessionID, client *ClientSession) bool { + moment := clientMoment + if client.LastMarked().Before(moment) { + s.removeClientSession(ctx, id) } - if sess.LastMarked().Before(moment) { - s.RemoveSession(ctx, id) + return true + }) + s.agents.Range(func(id tunnel.SessionID, agent *AgentSession) bool { + moment := agentMoment + if agent.LastMarked().Before(moment) { + s.removeAgentSession(ctx, id) } return true }) @@ -400,42 +415,42 @@ func (s *state) ExpireSessions(ctx context.Context, clientMoment, agentMoment ti // SessionDone returns a channel that is closed when the session with the given ID terminates. If // there is no such currently-live session, then an already-closed channel is returned. -func (s *state) SessionDone(id string) (<-chan struct{}, error) { - sess, ok := s.sessions.Load(id) - if !ok { - return nil, status.Errorf(codes.NotFound, "session %q not found", id) +func (s *state) SessionDone(id tunnel.SessionID) (<-chan struct{}, error) { + if cs, ok := s.clients.Load(id); ok { + return cs.Done(), nil + } + if as, ok := s.agents.Load(id); ok { + return as.Done(), nil } - return sess.Done(), nil + return nil, status.Errorf(codes.NotFound, "session %q not found", id) } // Sessions: Clients /////////////////////////////////////////////////////////////////////////////// -func (s *state) AddClient(client *rpc.ClientInfo, now time.Time) string { +func (s *state) AddClient(client *rpc.ClientInfo, now time.Time) tunnel.SessionID { // Use non-sequential things (i.e., UUIDs, not just a counter) as the session ID, because // the session ID also exists in external systems (the client, SystemA), so it's confusing // (to both humans and computers) if the manager restarts and those existing session IDs // suddenly refer to different sessions. - sessionID := uuid.New().String() - return s.addClient(sessionID, client, now) + sessionID := tunnel.SessionID(uuid.New().String()) + s.addClient(sessionID, client, now) + return sessionID } // addClient is like AddClient, but takes a sessionID, for testing purposes. -func (s *state) addClient(sessionID string, client *rpc.ClientInfo, now time.Time) string { - s.mu.Lock() - if oldClient, hasConflict := s.clients.LoadOrStore(sessionID, client); hasConflict { - panic(fmt.Errorf("duplicate id %q, existing %+v, new %+v", sessionID, oldClient, client)) +func (s *state) addClient(id tunnel.SessionID, client *rpc.ClientInfo, now time.Time) { + cs := newClientSessionState(s.backgroundCtx, id, client, now) + if oldClient, hasConflict := s.clients.LoadOrStore(id, cs); hasConflict { + panic(fmt.Errorf("duplicate id %q, existing %+v, new %+v", id, oldClient, client)) } - s.sessions.Store(sessionID, newClientSessionState(s.backgroundCtx, now)) - s.mu.Unlock() - return sessionID } -func (s *state) GetClient(sessionID string) *rpc.ClientInfo { - ret, _ := s.clients.Load(sessionID) +func (s *state) GetClient(id tunnel.SessionID) *ClientSession { + ret, _ := s.clients.Load(id) return ret } -func (s *state) EachClient(f func(string, *rpc.ClientInfo) bool) { +func (s *state) EachClient(f func(tunnel.SessionID, *ClientSession) bool) { s.clients.Range(f) } @@ -452,7 +467,7 @@ func (s *state) CountIntercepts() int { } func (s *state) CountSessions() int { - return s.sessions.Size() + return s.CountAgents() + s.CountClients() } func (s *state) CountTunnels() int { @@ -469,16 +484,16 @@ func (s *state) CountTunnelEgress() uint64 { // Sessions: Agents //////////////////////////////////////////////////////////////////////////////// -func (s *state) AddAgent(ctx context.Context, agent *rpc.AgentInfo, now time.Time) (string, error) { +func (s *state) AddAgent(ctx context.Context, agent *rpc.AgentInfo, now time.Time) (tunnel.SessionID, error) { if mutator.GetMap(ctx).IsInactive(types.UID(agent.PodUid)) { return "", status.Error(codes.Aborted, "inactivated pod") } - sessionID := AgentSessionIDPrefix + agent.PodUid - if oldAgent, hasConflict := s.agents.LoadOrStore(sessionID, agent); hasConflict { - return "", status.Error(codes.AlreadyExists, fmt.Sprintf("duplicate id %q, existing %+v, new %+v", sessionID, oldAgent, agent)) + id := tunnel.SessionID(AgentSessionIDPrefix + agent.PodUid) + as := newAgentSessionState(s.backgroundCtx, id, agent, now) + if oldAgent, hasConflict := s.agents.LoadOrStore(id, as); hasConflict { + return "", status.Error(codes.AlreadyExists, fmt.Sprintf("duplicate id %q, existing %+v, new %+v", id, oldAgent, agent)) } - s.sessions.Store(sessionID, newAgentSessionState(s.backgroundCtx, now)) s.intercepts.Range(func(interceptID string, intercept *Intercept) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED { return true @@ -499,11 +514,11 @@ func (s *state) AddAgent(ctx context.Context, agent *rpc.AgentInfo, now time.Tim } return true }) - return sessionID, nil + return id, nil } -func (s *state) GetAgent(sessionID string) *rpc.AgentInfo { - if ret, ok := s.agents.Load(sessionID); ok { +func (s *state) GetAgent(id tunnel.SessionID) *AgentSession { + if ret, ok := s.agents.Load(id); ok { if !mutator.GetMap(s.backgroundCtx).IsInactive(types.UID(ret.PodUid)) { return ret } @@ -511,25 +526,25 @@ func (s *state) GetAgent(sessionID string) *rpc.AgentInfo { return nil } -func (s *state) EachAgent(f func(string, *rpc.AgentInfo) bool) { +func (s *state) EachAgent(f func(tunnel.SessionID, *AgentSession) bool) { m := mutator.GetMap(s.backgroundCtx) - s.agents.Range(func(si string, ag *rpc.AgentInfo) bool { + s.agents.Range(func(id tunnel.SessionID, ag *AgentSession) bool { if !m.IsInactive(types.UID(ag.PodUid)) { - return f(si, ag) + return f(id, ag) } return true }) } -func (s *state) LoadMatchingAgents(f func(string, *rpc.AgentInfo) bool) map[string]*rpc.AgentInfo { +func (s *state) LoadMatchingAgents(f func(tunnel.SessionID, *AgentSession) bool) map[tunnel.SessionID]*AgentSession { m := mutator.GetMap(s.backgroundCtx) - return s.agents.LoadMatching(func(s string, ai *rpc.AgentInfo) bool { - return !m.IsInactive(types.UID(ai.PodUid)) && f(s, ai) + return s.agents.LoadMatching(func(id tunnel.SessionID, ai *AgentSession) bool { + return !m.IsInactive(types.UID(ai.PodUid)) && f(id, ai) }) } func (s *state) HasAgent(name, namespace string) (ok bool) { - s.EachAgent(func(_ string, ai *rpc.AgentInfo) bool { + s.EachAgent(func(_ tunnel.SessionID, ai *AgentSession) bool { if ai.Name == name && ai.Namespace == namespace { ok = true return false @@ -541,8 +556,8 @@ func (s *state) HasAgent(name, namespace string) (ok bool) { func (s *state) WatchAgents( ctx context.Context, - filter func(sessionID string, agent *rpc.AgentInfo) bool, -) <-chan map[string]*rpc.AgentInfo { + filter func(tunnel.SessionID, *AgentSession) bool, +) <-chan map[tunnel.SessionID]*AgentSession { return s.agents.Subscribe(ctx.Done(), filter) } @@ -563,8 +578,8 @@ func (s *state) WatchWorkloads(ctx context.Context, ns string) (ch <-chan []work // Intercepts ////////////////////////////////////////////////////////////////////////////////////// // getAgentsInNamespace returns the session IDs the agents in the given namespace. -func (s *state) getAgentsInNamespace(namespace string) map[string]*rpc.AgentInfo { - return s.LoadMatchingAgents(func(_ string, ai *rpc.AgentInfo) bool { +func (s *state) getAgentsInNamespace(namespace string) map[tunnel.SessionID]*AgentSession { + return s.LoadMatchingAgents(func(_ tunnel.SessionID, ai *AgentSession) bool { return ai.Namespace == namespace }) } @@ -603,10 +618,10 @@ func (s *state) RemoveIntercept(ctx context.Context, interceptID string) { } func (s *state) UninstallAgents(ctx context.Context, ur *rpc.UninstallAgentsRequest) error { - sessionID := ur.GetSessionInfo().GetSessionId() - clientInfo := s.GetClient(sessionID) + id := tunnel.SessionID(ur.GetSessionInfo().GetSessionId()) + clientInfo := s.GetClient(id) if clientInfo == nil { - return status.Errorf(codes.NotFound, "Client session %q not found", sessionID) + return status.Errorf(codes.NotFound, "Client session %q not found", id) } ns := clientInfo.GetNamespace() mm := mutator.GetMap(ctx) @@ -648,37 +663,61 @@ func (s *state) WatchIntercepts( } func (s *state) Tunnel(ctx context.Context, stream tunnel.Stream) error { - sessionID := stream.SessionID() - ss, ok := s.sessions.Load(sessionID) - if !ok { - return status.Errorf(codes.NotFound, "Session %q not found", sessionID) + id := stream.SessionID() + if cs, ok := s.clients.Load(id); ok { + return s.clientTunnel(ctx, cs, stream) } + if as, ok := s.agents.Load(id); ok { + return s.agentTunnel(ctx, as, stream) + } + return status.Errorf(codes.NotFound, "Session %q not found", id) +} +func (s *state) agentTunnel(ctx context.Context, agent *AgentSession, stream tunnel.Stream) error { var scm *SessionConsumptionMetrics - switch sst := ss.(type) { - case *agentSessionState: - // If it's an agent, find the associated clientSessionState. - if clientSessionID := sst.AwaitingBidiMapOwnerSessionID(stream); clientSessionID != "" { - s.mu.RLock() - as, ok := s.sessions.Load(clientSessionID) // get awaiting state - s.mu.RUnlock() - if ok { // if found - if css, isClient := as.(*clientSessionState); isClient { - scm = css.ConsumptionMetrics() - } - } + + // If it's an agent, find the associated ClientSession. + if clientSessionID := agent.AwaitingBidiMapOwnerSessionID(stream); clientSessionID != "" { + cs, ok := s.clients.Load(clientSessionID) // get awaiting state + if ok { // if found + scm = cs.ConsumptionMetrics() } - case *clientSessionState: - scm = sst.ConsumptionMetrics() - default: } - bidiPipe, err := ss.OnConnect(ctx, stream, &s.tunnelCounter, scm) + if bidiPipe, err := agent.OnConnect(ctx, stream, &s.tunnelCounter, scm); err != nil { + return err + } else if bidiPipe != nil { + // A peer awaited this stream. Wait for the bidiPipe to finish + <-bidiPipe.Done() + return nil + } + + // A traffic-agent must always extend the tunnel to the client that it is currently intercepted + // by, and hence, start by sending the sessionID of that client on the tunnel. + + // Obtain the desired client session + m, err := stream.Receive(ctx) if err != nil { + return status.Errorf(codes.FailedPrecondition, "failed to read first message from agent tunnel %q: %v", agent.PodName, err) + } + if m.Code() != tunnel.Session { + return status.Errorf(codes.FailedPrecondition, "unable to read ClientSession from agent %q", agent.PodName) + } + if peerSession, ok := s.clients.Load(tunnel.GetSession(m)); ok { + endPoint, err := peerSession.EstablishBidiPipe(ctx, stream) + if err == nil { + <-endPoint.Done() + } return err } + return nil +} - if bidiPipe != nil { +func (s *state) clientTunnel(ctx context.Context, client *ClientSession, stream tunnel.Stream) error { + scm := client.ConsumptionMetrics() + if bidiPipe, err := client.OnConnect(ctx, stream, &s.tunnelCounter, scm); err != nil { + return err + } else if bidiPipe != nil { // A peer awaited this stream. Wait for the bidiPipe to finish <-bidiPipe.Done() return nil @@ -688,99 +727,65 @@ func (s *state) Tunnel(ctx context.Context, stream tunnel.Stream) error { // // A client will want to extend the tunnel to a dialer in an intercepted traffic-agent or, if no // intercept is active, to a dialer in that namespace. - // - // A traffic-agent must always extend the tunnel to the client that it is currently intercepted - // by, and hence, start by sending the sessionID of that client on the tunnel. - var peerSession SessionState - if _, ok := ss.(*agentSessionState); ok { - // traffic-agent, so obtain the desired client session - m, err := stream.Receive(ctx) - if err != nil { - return status.Errorf(codes.FailedPrecondition, "failed to read first message from agent tunnel %q: %v", sessionID, err) - } - if m.Code() != tunnel.Session { - return status.Errorf(codes.FailedPrecondition, "unable to read ClientSession from agent %q", sessionID) - } - peerID := tunnel.GetSession(m) - peerSession, _ = s.sessions.Load(peerID) - } else { - peerSession, err = s.getAgentForDial(ctx, sessionID, stream.ID().DestinationAddr()) - if err != nil { - return err + if peerSession := s.getAgentForDial(ctx, client, stream.ID().DestinationAddr()); peerSession != nil { + endPoint, err := peerSession.EstablishBidiPipe(ctx, stream) + if err == nil { + <-endPoint.Done() } + return err } - var endPoint tunnel.Endpoint - if peerSession != nil { - var err error - if endPoint, err = peerSession.EstablishBidiPipe(ctx, stream); err != nil { - return err - } - } else { - if css, isClient := ss.(*clientSessionState); isClient { - scm = css.ConsumptionMetrics() - } - endPoint = tunnel.NewDialer(stream, func() {}, scm.FromClientBytes, scm.ToClientBytes) - endPoint.Start(ctx) - } + // No peerSession exists, so use the traffic-manager itself for the dial. + endPoint := tunnel.NewDialer(stream, func() {}, scm.FromClientBytes, scm.ToClientBytes) + endPoint.Start(ctx) <-endPoint.Done() return nil } -func (s *state) getAgentForDial(ctx context.Context, clientSessionID string, podIP netip.Addr) (SessionState, error) { - agentKey, err := s.getAgentIdForDial(ctx, clientSessionID, podIP) - if err != nil || agentKey == "" { - return nil, err - } - agent, _ := s.sessions.Load(agentKey) - return agent, nil -} - -func (s *state) getAgentIdForDial(ctx context.Context, clientSessionID string, podIP netip.Addr) (string, error) { +func (s *state) getAgentForDial(ctx context.Context, client *ClientSession, podIP netip.Addr) *AgentSession { // An agent with a podIO matching the given podIP has precedence - agents := s.LoadMatchingAgents(func(key string, ai *rpc.AgentInfo) bool { + agents := s.LoadMatchingAgents(func(key tunnel.SessionID, ai *AgentSession) bool { if aip, err := netip.ParseAddr(ai.PodIp); err == nil { return podIP == aip } return false }) - for agentID := range agents { + for _, agent := range agents { dlog.Debugf(ctx, "selecting agent for dial based on podIP %s", podIP) - return agentID, nil + return agent } - client, ok := s.clients.Load(clientSessionID) - if !ok { - return "", status.Errorf(codes.NotFound, "session %q not found", clientSessionID) - } env := managerutil.GetEnv(ctx) if env.ManagerNamespace == client.Namespace { // Traffic manager will do just fine dlog.Debugf(ctx, "selecting traffic-manager for dial, because it's in namespace %q", client.Namespace) - return "", nil + return nil } // Any agent that is currently intercepted by the client has precedence. - for agentID := range s.getAgentsInterceptedByClient(clientSessionID) { - dlog.Debugf(ctx, "selecting intercepted agent %q for dial", agentID) - return agentID, nil + for _, agent := range s.getAgentsInterceptedByClient(client.ID()) { + dlog.Debugf(ctx, "selecting intercepted agent %q for dial", agent.PodName) + return agent } // Any agent from the same namespace will do. - for agentID := range s.getAgentsInNamespace(client.Namespace) { - dlog.Debugf(ctx, "selecting agent %q for dial based on namespace %q", agentID, client.Namespace) - return agentID, nil + for _, agent := range s.getAgentsInNamespace(client.Namespace) { + dlog.Debugf(ctx, "selecting agent %q for dial based on namespace %q", agent.PodName, client.Namespace) + return agent } // Best effort is to use the traffic-manager. // TODO: Add a pod that can dial from the correct namespace dlog.Debugf(ctx, "selecting traffic-manager for dial, even though it's not in namespace %q", client.Namespace) - return "", nil + return nil } -func (s *state) WatchDial(sessionID string) <-chan *rpc.DialRequest { - if ss, ok := s.sessions.Load(sessionID); ok { - return ss.Dials() +func (s *state) WatchDial(id tunnel.SessionID) <-chan *rpc.DialRequest { + if cs := s.GetClient(id); cs != nil { + return cs.Dials() + } + if as := s.GetAgent(id); as != nil { + return as.Dials() } return nil } @@ -849,7 +854,7 @@ func (s *state) SetAllClientSessionsFinalizer(finalizer allClientSessionsFinaliz s.allClientSessionsFinalizer = finalizer } -func (s *state) allClientSessionsFinalizerCall(client *rpc.ClientInfo) { +func (s *state) allClientSessionsFinalizerCall(client *ClientSession) { if s.allClientSessionsFinalizer != nil { s.allClientSessionsFinalizer(client) } @@ -859,7 +864,7 @@ func (s *state) SetAllInterceptsFinalizer(finalizer allInterceptsFinalizer) { s.allInterceptsFinalizer = finalizer } -func (s *state) allInterceptsFinalizerCall(client *rpc.ClientInfo, workload *string) { +func (s *state) allInterceptsFinalizerCall(client *ClientSession, workload *string) { if s.allInterceptsFinalizer != nil { s.allInterceptsFinalizer(client, workload) } diff --git a/cmd/traffic/cmd/manager/state/state_test.go b/cmd/traffic/cmd/manager/state/state_test.go index 7df66e8d1f..c247985e14 100644 --- a/cmd/traffic/cmd/manager/state/state_test.go +++ b/cmd/traffic/cmd/manager/state/state_test.go @@ -16,6 +16,7 @@ import ( testdata "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/test" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/watchable" "github.com/telepresenceio/telepresence/v2/pkg/log" + "github.com/telepresenceio/telepresence/v2/pkg/tunnel" "github.com/telepresenceio/telepresence/v2/pkg/workload" ) @@ -31,10 +32,9 @@ func (s *suiteState) SetupTest() { s.state = &state{ backgroundCtx: s.ctx, intercepts: watchable.NewMap[string, *Intercept](interceptEqual, time.Millisecond), - agents: watchable.NewMap[string, *manager.AgentInfo](agentsEqual, time.Millisecond), - clients: xsync.NewMapOf[string, *manager.ClientInfo](), + agents: watchable.NewMap[tunnel.SessionID, *AgentSession](agentsEqual, time.Millisecond), + clients: xsync.NewMapOf[tunnel.SessionID, *ClientSession](), workloadWatchers: xsync.NewMapOf[string, workload.Watcher](), - sessions: xsync.NewMapOf[string, SessionState](), timedLogLevel: log.NewTimedLevel("debug", log.SetLevel), llSubs: newLoglevelSubscribers(), } @@ -50,25 +50,6 @@ func (fc *FakeClock) Now() time.Time { return base.Add(offset) } -func getAllAgents(st *state) []*manager.AgentInfo { - agents := make([]*manager.AgentInfo, 0, st.agents.Size()) - st.agents.Range(func(_ string, a *manager.AgentInfo) bool { - agents = append(agents, a) - return true - }) - return agents -} - -func getAgentsByName(st *state, name, namespace string) (agents []*manager.AgentInfo) { - st.agents.Range(func(_ string, a *manager.AgentInfo) bool { - if a.Name == name && a.Namespace == namespace { - agents = append(agents, a) - } - return true - }) - return agents -} - func (s *suiteState) TestStateInternal() { ctx := context.Background() @@ -97,36 +78,10 @@ func (s *suiteState) TestStateInternal() { d2, err := st.AddAgent(ctx, demoAgent2, clock.Now()) require.NoError(t, err) - a.Equal(helloAgent, st.GetAgent(h)) - a.Equal(helloProAgent, st.GetAgent(hp)) - a.Equal(demoAgent1, st.GetAgent(d1)) - a.Equal(demoAgent2, st.GetAgent(d2)) - - agents := getAllAgents(st) - a.Len(agents, 4) - a.Contains(agents, helloAgent) - a.Contains(agents, helloProAgent) - a.Contains(agents, demoAgent1) - a.Contains(agents, demoAgent2) - - agents = getAgentsByName(st, "hello", "default") - a.Len(agents, 1) - a.Contains(agents, helloAgent) - - agents = getAgentsByName(st, "hello-pro", "default") - a.Len(agents, 1) - a.Contains(agents, helloProAgent) - - agents = getAgentsByName(st, "demo", "default") - a.Len(agents, 2) - a.Contains(agents, demoAgent1) - a.Contains(agents, demoAgent2) - - agents = getAgentsByName(st, "does-not-exist", "default") - a.Len(agents, 0) - - agents = getAgentsByName(st, "hello", "does-not-exist") - a.Len(agents, 0) + a.Equal(helloAgent, st.GetAgent(h).AgentInfo) + a.Equal(helloProAgent, st.GetAgent(hp).AgentInfo) + a.Equal(demoAgent1, st.GetAgent(d1).AgentInfo) + a.Equal(demoAgent2, st.GetAgent(d2).AgentInfo) }) s.T().Run("presence-redundant", func(t *testing.T) { @@ -145,12 +100,12 @@ func (s *suiteState) TestStateInternal() { a.NotNil(s.GetClient(c3)) a.Nil(s.GetClient("asdf")) - a.Equal(testClients["alice"], s.GetClient(c1)) + a.Equal(testClients["alice"], s.GetClient(c1).ClientInfo) clock.When = 10 - a.True(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: c1}}, clock.Now())) - a.True(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: c2}}, clock.Now())) + a.True(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: string(c1)}}, clock.Now())) + a.True(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: string(c2)}}, clock.Now())) a.False(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: "asdf"}}, clock.Now())) moment := epoch.Add(5 * time.Second) @@ -162,9 +117,9 @@ func (s *suiteState) TestStateInternal() { clock.When = 20 - a.True(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: c1}}, clock.Now())) - a.True(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: c2}}, clock.Now())) - a.False(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: c3}}, clock.Now())) + a.True(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: string(c1)}}, clock.Now())) + a.True(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: string(c2)}}, clock.Now())) + a.False(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: string(c3)}}, clock.Now())) moment = epoch.Add(5 * time.Second) s.ExpireSessions(ctx, moment, moment) @@ -179,9 +134,9 @@ func (s *suiteState) TestStateInternal() { a.Nil(s.GetClient(c2)) a.Nil(s.GetClient(c3)) - a.True(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: c1}}, clock.Now())) - a.False(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: c2}}, clock.Now())) - a.False(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: c3}}, clock.Now())) + a.True(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: string(c1)}}, clock.Now())) + a.False(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: string(c2)}}, clock.Now())) + a.False(s.MarkSession(&manager.RemainRequest{Session: &manager.SessionInfo{SessionId: string(c3)}}, clock.Now())) }) } @@ -199,21 +154,33 @@ func (s *suiteState) TestAddClient() { }, now) // then - assert.Equal(s.T(), 1, s.state.sessions.Size()) + assert.Equal(s.T(), 1, s.state.clients.Size()) } func (s *suiteState) TestRemoveSession() { // given now := time.Now() - s.state.sessions.Store("session-1", newClientSessionState(s.ctx, now)) - s.state.sessions.Store("session-2", newAgentSessionState(s.ctx, now)) + s1 := s.state.AddClient(&manager.ClientInfo{ + Name: "my-client", + InstallId: "1234", + Product: "5668", + Version: "2.14.2", + ApiKey: "xxxx", + }, now) + s2 := s.state.AddClient(&manager.ClientInfo{ + Name: "your-client", + InstallId: "5678", + Product: "5668", + Version: "2.14.2", + ApiKey: "xxxx", + }, now) - // when - s.state.RemoveSession(s.ctx, "session-1") - s.state.RemoveSession(s.ctx, "session-2") // won't fail trying to delete consumption. + assert.Equal(s.T(), s.state.CountSessions(), 2) - // then - assert.Equal(s.T(), s.state.sessions.Size(), 0) + s.state.RemoveSession(s.ctx, s1) + s.state.RemoveSession(s.ctx, s2) // won't fail trying to delete consumption. + + assert.Equal(s.T(), s.state.CountSessions(), 0) } func TestSuiteState(testing *testing.T) { diff --git a/cmd/traffic/cmd/manager/state/workload_info_watcher.go b/cmd/traffic/cmd/manager/state/workload_info_watcher.go index 1d84c48889..3febc4df6b 100644 --- a/cmd/traffic/cmd/manager/state/workload_info_watcher.go +++ b/cmd/traffic/cmd/manager/state/workload_info_watcher.go @@ -15,6 +15,7 @@ import ( "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/mutator" "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" + "github.com/telepresenceio/telepresence/v2/pkg/tunnel" "github.com/telepresenceio/telepresence/v2/pkg/workload" ) @@ -24,18 +25,18 @@ type WorkloadInfoWatcher interface { type workloadInfoWatcher struct { State - clientSession string + clientSession tunnel.SessionID namespace string stream rpc.Manager_WatchWorkloadsServer workloadEvents map[string]*rpc.WorkloadEvent lastEvents map[string]*rpc.WorkloadEvent - agentInfos map[string]*rpc.AgentInfo + agentInfos map[tunnel.SessionID]*AgentSession interceptInfos map[string]*Intercept start time.Time ticker *time.Ticker } -func (s *state) NewWorkloadInfoWatcher(clientSession, namespace string) WorkloadInfoWatcher { +func (s *state) NewWorkloadInfoWatcher(clientSession tunnel.SessionID, namespace string) WorkloadInfoWatcher { return &workloadInfoWatcher{ State: s, clientSession: clientSession, @@ -68,7 +69,7 @@ func (wf *workloadInfoWatcher) Watch(ctx context.Context, stream rpc.Manager_Wat return err } - agentsCh := wf.WatchAgents(ctx, func(_ string, info *rpc.AgentInfo) bool { + agentsCh := wf.WatchAgents(ctx, func(_ tunnel.SessionID, info *AgentSession) bool { return info.Namespace == wf.namespace }) @@ -241,7 +242,7 @@ func (wf *workloadInfoWatcher) handleWorkloadsSnapshot(ctx context.Context, wes } } -func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[string]*rpc.AgentInfo) { +func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[tunnel.SessionID]*AgentSession) { oldAgentInfos := wf.agentInfos wf.agentInfos = ais m := mutator.GetMap(ctx) diff --git a/pkg/client/agentpf/clients.go b/pkg/client/agentpf/clients.go index 4b507b6ab9..78310c6da8 100644 --- a/pkg/client/agentpf/clients.go +++ b/pkg/client/agentpf/clients.go @@ -228,7 +228,7 @@ func (ac *client) startDialWatcherReady(ctx context.Context) error { ac.Unlock() go func() { - err := tunnel.DialWaitLoop(ctx, tunnel.AgentProvider(ac.cli), watcher, ac.session.SessionId) + err := tunnel.DialWaitLoop(ctx, tunnel.AgentProvider(ac.cli), watcher, tunnel.SessionID(ac.session.SessionId)) if err != nil { dlog.Error(ctx, err) } diff --git a/pkg/client/remotefs/bridge.go b/pkg/client/remotefs/bridge.go index db13e359c5..dd4739263f 100644 --- a/pkg/client/remotefs/bridge.go +++ b/pkg/client/remotefs/bridge.go @@ -16,11 +16,11 @@ import ( type bridgeMounter struct { localPort uint16 - sessionID string + sessionID tunnel.SessionID managerClient manager.ManagerClient } -func NewBridgeMounter(sessionID string, managerClient manager.ManagerClient, localPort uint16) Mounter { +func NewBridgeMounter(sessionID tunnel.SessionID, managerClient manager.ManagerClient, localPort uint16) Mounter { return &bridgeMounter{ localPort: localPort, sessionID: sessionID, diff --git a/pkg/client/rootd/stream_creator.go b/pkg/client/rootd/stream_creator.go index 929be1729c..06d9283d43 100644 --- a/pkg/client/rootd/stream_creator.go +++ b/pkg/client/rootd/stream_creator.go @@ -59,7 +59,7 @@ func (s *Session) streamCreator(ctx context.Context) tunnel.StreamCreator { if s.isForDNS(destAddr, id.DestinationPort()) { pipeId := tunnel.NewConnID(p, id.Source(), s.dnsLocalAddr.AddrPort()) dlog.Tracef(c, "Intercept DNS %s to %s", id, pipeId.Destination()) - from, to := tunnel.NewPipe(pipeId, s.session.SessionId) + from, to := tunnel.NewPipe(pipeId, tunnel.SessionID(s.session.SessionId)) tunnel.NewDialerTTL(to, func() {}, dnsConnTTL, nil, nil).Start(c) return from, nil } @@ -128,7 +128,8 @@ func (s *Session) streamCreator(ctx context.Context) tunnel.StreamCreator { } tc := client.GetConfig(c).Timeouts() - return tunnel.NewClientStream(c, ct, id, s.session.SessionId, tc.Get(client.TimeoutRoundtripLatency), tc.Get(client.TimeoutEndpointDial)) + return tunnel.NewClientStream( + c, ct, id, tunnel.SessionID(s.session.SessionId), tc.Get(client.TimeoutRoundtripLatency), tc.Get(client.TimeoutEndpointDial)) } } diff --git a/pkg/client/userd/trafficmgr/dial_request.go b/pkg/client/userd/trafficmgr/dial_request.go index d1cc20b6b9..9d79e0ba16 100644 --- a/pkg/client/userd/trafficmgr/dial_request.go +++ b/pkg/client/userd/trafficmgr/dial_request.go @@ -16,5 +16,5 @@ func (s *session) _dialRequestWatcher(ctx context.Context) error { if err != nil { return err } - return tunnel.DialWaitLoop(ctx, tunnel.ManagerProvider(s.managerClient), dialerStream, s.sessionInfo.SessionId) + return tunnel.DialWaitLoop(ctx, tunnel.ManagerProvider(s.managerClient), dialerStream, tunnel.SessionID(s.sessionInfo.SessionId)) } diff --git a/pkg/client/userd/trafficmgr/mount.go b/pkg/client/userd/trafficmgr/mount.go index 725e65e2c0..daeb6c89e4 100644 --- a/pkg/client/userd/trafficmgr/mount.go +++ b/pkg/client/userd/trafficmgr/mount.go @@ -14,6 +14,7 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/client" "github.com/telepresenceio/telepresence/v2/pkg/client/remotefs" "github.com/telepresenceio/telepresence/v2/pkg/client/userd" + "github.com/telepresenceio/telepresence/v2/pkg/tunnel" ) func (pa *podAccess) shouldMount() bool { @@ -61,7 +62,7 @@ func (pa *podAccess) startMount(ctx context.Context, iceptWG, podWG *sync.WaitGr switch { case pa.localMountPort != 0: session := userd.GetSession(ctx) - m = remotefs.NewBridgeMounter(session.SessionInfo().SessionId, session.ManagerClient(), uint16(pa.localMountPort)) + m = remotefs.NewBridgeMounter(tunnel.SessionID(session.SessionInfo().SessionId), session.ManagerClient(), uint16(pa.localMountPort)) case useFtp: m = remotefs.NewFTPMounter(fuseftp, iceptWG) default: diff --git a/pkg/forwarder/tcp.go b/pkg/forwarder/tcp.go index b4d63ffdaa..fd920a76f5 100644 --- a/pkg/forwarder/tcp.go +++ b/pkg/forwarder/tcp.go @@ -157,13 +157,13 @@ func (f *tcp) interceptConn(ctx context.Context, conn net.Conn, iCept *manager.I return f.rerouteConn( ctx, conn, - iCept.ClientSession.SessionId, + tunnel.SessionID(iCept.ClientSession.SessionId), netip.AddrPortFrom(iputil.Parse(spec.TargetHost), uint16(spec.TargetPort)), time.Duration(spec.RoundtripLatency), time.Duration(spec.DialTimeout)) } -func (f *tcp) rerouteConn(ctx context.Context, conn net.Conn, clientSession string, dst netip.AddrPort, latency, timeout time.Duration) error { +func (f *tcp) rerouteConn(ctx context.Context, conn net.Conn, clientSession tunnel.SessionID, dst netip.AddrPort, latency, timeout time.Duration) error { srcAddr := conn.RemoteAddr() dlog.Debugf(ctx, "Accept got connection from %s", srcAddr) defer dlog.Debugf(ctx, "Done serving connection from %s", srcAddr) @@ -194,7 +194,7 @@ func (f *tcp) rerouteConn(ctx context.Context, conn net.Conn, clientSession stri <-d.Done() sp.ReportMetrics(ctx, &manager.TunnelMetrics{ - ClientSessionId: clientSession, + ClientSessionId: string(clientSession), IngressBytes: ingressBytes.GetValue(), EgressBytes: egressBytes.GetValue(), }) diff --git a/pkg/forwarder/udp.go b/pkg/forwarder/udp.go index 067aa6bc8d..41dbbe6878 100644 --- a/pkg/forwarder/udp.go +++ b/pkg/forwarder/udp.go @@ -209,7 +209,7 @@ func (f *udp) interceptConn(ctx context.Context, conn *net.UDPConn, iCept *manag f.mu.Lock() sp := f.streamProvider f.mu.Unlock() - return sp.CreateClientStream(ctx, iCept.ClientSession.SessionId, id, time.Duration(spec.RoundtripLatency), time.Duration(spec.DialTimeout)) + return sp.CreateClientStream(ctx, tunnel.SessionID(iCept.ClientSession.SessionId), id, time.Duration(spec.RoundtripLatency), time.Duration(spec.DialTimeout)) }) d.Start(ctx) <-d.Done() diff --git a/pkg/tunnel/client_stream.go b/pkg/tunnel/client_stream.go index d13cb7d385..8599d6deea 100644 --- a/pkg/tunnel/client_stream.go +++ b/pkg/tunnel/client_stream.go @@ -12,7 +12,7 @@ type GRPCClientStream interface { CloseSend() error } -func NewClientStream(ctx context.Context, grpcStream GRPCClientStream, id ConnID, sessionID string, callDelay, dialTimeout time.Duration) (Stream, error) { +func NewClientStream(ctx context.Context, grpcStream GRPCClientStream, id ConnID, sessionID SessionID, callDelay, dialTimeout time.Duration) (Stream, error) { s := &clientStream{stream: newStream("CLI", grpcStream)} s.id = id s.roundtripLatency = callDelay diff --git a/pkg/tunnel/dialer.go b/pkg/tunnel/dialer.go index 81b9cf3ebc..5d00c22ebe 100644 --- a/pkg/tunnel/dialer.go +++ b/pkg/tunnel/dialer.go @@ -333,7 +333,7 @@ func DialWaitLoop( ctx context.Context, tunnelProvider Provider, dialStream rpc.Manager_WatchDialClient, - sessionID string, + sessionID SessionID, ) error { // create ctx to cleanup leftover dialRespond if waitloop dies ctx, cancel := context.WithCancel(ctx) @@ -359,7 +359,7 @@ func DialWaitLoop( return nil } -func dialRespond(ctx context.Context, tunnelProvider Provider, dr *rpc.DialRequest, sessionID string) { +func dialRespond(ctx context.Context, tunnelProvider Provider, dr *rpc.DialRequest, sessionID SessionID) { id := ConnID(dr.ConnId) ctx, cancel := context.WithCancel(ctx) mt, err := tunnelProvider.Tunnel(ctx) diff --git a/pkg/tunnel/message.go b/pkg/tunnel/message.go index 595b405ba3..34b1c2bfa0 100644 --- a/pkg/tunnel/message.go +++ b/pkg/tunnel/message.go @@ -119,7 +119,7 @@ func NewMessage(code MessageCode, payload []byte) Message { return msg{byte(code)} } -func StreamInfoMessage(id ConnID, sessionID string, callDelay, dialTimeout time.Duration) Message { +func StreamInfoMessage(id ConnID, sessionID SessionID, callDelay, dialTimeout time.Duration) Message { b := bytes.Buffer{} b.WriteByte(byte(streamInfo)) @@ -151,12 +151,12 @@ func StreamOKMessage() Message { return m[:n+1] } -func SessionMessage(sessionID string) Message { +func SessionMessage(sessionID SessionID) Message { return NewMessage(Session, []byte(sessionID)) } -func GetSession(m Message) string { - return string(m.Payload()) +func GetSession(m Message) SessionID { + return SessionID(m.Payload()) } func makeMessage(code MessageCode, payloadLength int) msg { @@ -211,6 +211,6 @@ func setConnectInfo(m Message, s *stream) error { return errMalformedConnect } pl = pl[n:] - s.sessionID = string(pl[:v]) + s.sessionID = SessionID(pl[:v]) return nil } diff --git a/pkg/tunnel/metrics.go b/pkg/tunnel/metrics.go index 658776a1b5..c1bf62c2b7 100644 --- a/pkg/tunnel/metrics.go +++ b/pkg/tunnel/metrics.go @@ -9,29 +9,23 @@ import ( "github.com/telepresenceio/telepresence/rpc/v2/manager" ) -type StreamMetrics struct { - ClientSessionID string - IngressBytes *CounterProbe - EgressBytes *CounterProbe -} - type StreamProvider interface { - CreateClientStream(ctx context.Context, clientSessionID string, id ConnID, roundTripLatency, dialTimeout time.Duration) (Stream, error) + CreateClientStream(ctx context.Context, clientSessionID SessionID, id ConnID, roundTripLatency, dialTimeout time.Duration) (Stream, error) } type ClientStreamProvider interface { - CreateClientStream(ctx context.Context, clientSessionID string, id ConnID, roundTripLatency, dialTimeout time.Duration) (Stream, error) + CreateClientStream(ctx context.Context, clientSessionID SessionID, id ConnID, roundTripLatency, dialTimeout time.Duration) (Stream, error) ReportMetrics(ctx context.Context, metrics *manager.TunnelMetrics) } type TrafficManagerStreamProvider struct { Manager manager.ManagerClient - AgentSessionID string + AgentSessionID SessionID } func (sp *TrafficManagerStreamProvider) CreateClientStream( ctx context.Context, - clientSessionID string, + clientSessionID SessionID, id ConnID, roundTripLatency, dialTimeout time.Duration, diff --git a/pkg/tunnel/pipe.go b/pkg/tunnel/pipe.go index 6c6a0b083c..7c98d80c66 100644 --- a/pkg/tunnel/pipe.go +++ b/pkg/tunnel/pipe.go @@ -7,7 +7,7 @@ import ( ) // NewPipe creates a pair of Streams connected using two channels. -func NewPipe(id ConnID, sessionID string) (Stream, Stream) { +func NewPipe(id ConnID, sessionID SessionID) (Stream, Stream) { out := make(chan Message, 1) in := make(chan Message, 1) return &channelStream{ @@ -28,7 +28,7 @@ func NewPipe(id ConnID, sessionID string) (Stream, Stream) { type channelStream struct { id ConnID tag string - sid string + sid SessionID recvCh <-chan Message sendCh chan<- Message } @@ -70,7 +70,7 @@ func (s channelStream) PeerVersion() uint16 { return 2 } -func (s channelStream) SessionID() string { +func (s channelStream) SessionID() SessionID { return s.sid } diff --git a/pkg/tunnel/stream.go b/pkg/tunnel/stream.go index 298e0f5791..a024f8b6b1 100644 --- a/pkg/tunnel/stream.go +++ b/pkg/tunnel/stream.go @@ -22,6 +22,8 @@ import ( // 1 used MuxTunnel instead of one tunnel per connection. const Version = uint16(2) +type SessionID string + // Endpoint is an endpoint for a Stream such as a Dialer or a bidirectional pipe. type Endpoint interface { Start(ctx context.Context) @@ -63,7 +65,7 @@ type Stream interface { Send(context.Context, Message) error CloseSend(ctx context.Context) error PeerVersion() uint16 - SessionID() string + SessionID() SessionID DialTimeout() time.Duration RoundtripLatency() time.Duration } @@ -180,7 +182,7 @@ type stream struct { id ConnID dialTimeout time.Duration roundtripLatency time.Duration - sessionID string + sessionID SessionID tag string syncRatio uint32 // send and check sync after each syncRatio message ackWindow uint32 // maximum permitted difference between sent and received ack @@ -211,7 +213,7 @@ func (s *stream) RoundtripLatency() time.Duration { return s.roundtripLatency } -func (s *stream) SessionID() string { +func (s *stream) SessionID() SessionID { return s.sessionID } diff --git a/pkg/tunnel/stream_test.go b/pkg/tunnel/stream_test.go index 96c5e1c01f..df6a924aa6 100644 --- a/pkg/tunnel/stream_test.go +++ b/pkg/tunnel/stream_test.go @@ -113,7 +113,7 @@ func TestStream_Connect(t *testing.T) { tunnel := newBidi(10, ctx.Done()) id := NewConnID(ipproto.TCP, netip.AddrPortFrom(netip.AddrFrom4([4]byte{127, 0, 0, 1}), 1001), netip.AddrPortFrom(netip.AddrFrom4([4]byte{192, 168, 0, 1}), 8080)) - si := uuid.New().String() + si := SessionID(uuid.New().String()) wg := sync.WaitGroup{} wg.Add(2) @@ -214,7 +214,7 @@ func TestStream_Xfer(t *testing.T) { defer cancel() id := NewConnID(ipproto.TCP, netip.AddrPortFrom(netip.AddrFrom4([4]byte{127, 0, 0, 1}), 1001), netip.AddrPortFrom(netip.AddrFrom4([4]byte{192, 168, 0, 1}), 8080)) - si := uuid.New().String() + si := SessionID(uuid.New().String()) b := make([]byte, 0x1000) for i := range b { b[i] = byte(i & 0xff) From 505406ef67b93eda9ae6f6c19d14fb01d13689c1 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Fri, 31 Jan 2025 18:32:53 +0100 Subject: [PATCH 37/61] Use a GRPC interceptor for logging. Signed-off-by: Thomas Hallgren --- DEPENDENCIES.md | 1 + build-aux/unparsable-packages.yaml | 4 + cmd/traffic/cmd/manager/manager.go | 8 - cmd/traffic/cmd/manager/mutator/watcher.go | 17 +- cmd/traffic/cmd/manager/service.go | 52 ++--- cmd/traffic/cmd/manager/state/state.go | 70 +++---- go.mod | 1 + go.sum | 2 + integration_test/argo_rollouts_test.go | 9 +- integration_test/podscaling_test.go | 8 + pkg/agentmap/discorvery.go | 8 +- pkg/client/userd/daemon/grpc.go | 228 ++++++++------------- pkg/client/userd/daemon/service.go | 1 - pkg/client/userd/service.go | 4 +- pkg/grpc/server/logging.go | 60 ++++++ pkg/grpc/server/server.go | 60 +++++- 16 files changed, 277 insertions(+), 256 deletions(-) create mode 100644 pkg/grpc/server/logging.go diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 1aeca46311..b1b9a21843 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -73,6 +73,7 @@ following Free and Open Source software: github.com/gorilla/websocket v1.5.3 2-clause BSD license github.com/gosuri/uitable v0.0.4 MIT license github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 MIT license + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0 Apache License 2.0 github.com/hashicorp/errwrap v1.1.0 Mozilla Public License 2.0 github.com/hashicorp/go-multierror v1.1.1 Mozilla Public License 2.0 github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb MIT license diff --git a/build-aux/unparsable-packages.yaml b/build-aux/unparsable-packages.yaml index ba8a19f744..9acce974ee 100644 --- a/build-aux/unparsable-packages.yaml +++ b/build-aux/unparsable-packages.yaml @@ -18,3 +18,7 @@ github.com/fclairamb/ftpserverlib: - MIT github.com/Masterminds/squirrel: - MIT +github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors: + - Apache-2.0 +github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging: + - Apache-2.0 diff --git a/cmd/traffic/cmd/manager/manager.go b/cmd/traffic/cmd/manager/manager.go index 0b646163af..1a086c94a0 100644 --- a/cmd/traffic/cmd/manager/manager.go +++ b/cmd/traffic/cmd/manager/manager.go @@ -280,14 +280,6 @@ func (s *service) serveHTTP(ctx context.Context) error { if mz, ok := env.MaxReceiveSize.AsInt64(); ok { opts = append(opts, grpc.MaxRecvMsgSize(int(mz))) } - - lg := dlog.StdLogger(ctx, dlog.MaxLogLevel(ctx)) - addr := iputil.JoinHostPort(host, port) - if host == "" { - lg.SetPrefix(fmt.Sprintf("grpc-api:%d", port)) - } else { - lg.SetPrefix(fmt.Sprintf("grpc-api %s", addr)) - } svc := server.New(ctx, opts...) s.self.RegisterServers(svc) return server.Serve(ctx, svc, l) diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index 0ef5f0e6a7..7cd0ca3623 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -2,6 +2,7 @@ package mutator import ( "context" + "errors" "fmt" "slices" "sync" @@ -650,13 +651,19 @@ func podList(ctx context.Context, kind k8sapi.Kind, name, namespace string) ([]* if !podIsRunning(pod) { continue } + if podKind, ok := pod.Labels[agentconfig.WorkloadKindLabel]; ok && !enabledWorkloads.Contains(k8sapi.Kind(podKind)) { + // Pod's label indicates a workload kind that has been disabled. + podsOfInterest = append(podsOfInterest, pod) + continue + } wl, err := agentmap.FindOwnerWorkload(ctx, k8sapi.Pod(pod), enabledWorkloads) - if err == nil { - if (kind == "" || wl.GetKind() == kind) && (name == "" || wl.GetName() == name) { - podsOfInterest = append(podsOfInterest, pod) + if err != nil { + var ew agentmap.WorkloadNotFoundError + if !errors.As(err, &ew) { + return nil, err } - } else { - dlog.Debugf(ctx, "error getting owner workload for pod %s.%s: %v", pod.Name, pod.Namespace, err) + } else if (kind == "" || wl.GetKind() == kind) && (name == "" || wl.GetName() == name) { + podsOfInterest = append(podsOfInterest, pod) } } return podsOfInterest, nil diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index 9317030578..a2d8469846 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -57,7 +57,6 @@ type Service interface { } type service struct { - ctx context.Context clock Clock id string state state.State @@ -106,7 +105,6 @@ func NewService(ctx context.Context, configWatcher config.Watcher) (Service, *dg dlog.Errorf(ctx, "unable to initialize agent injector: %v", err) } } - ret.ctx = ctx // These are context dependent so build them once the pool is up ret.clusterInfo = cluster.NewInfo(ctx) ret.state = state.NewStateFunc(ctx) @@ -159,7 +157,6 @@ func (s *service) GetAgentImageFQN(ctx context.Context, _ *empty.Empty) (*rpc.Ag } func (s *service) GetAgentConfig(ctx context.Context, request *rpc.AgentConfigRequest) (*rpc.AgentConfigResponse, error) { - dlog.Debug(ctx, "GetAgentConfig called") ctx = managerutil.WithSessionInfo(ctx, request.Session) sessionID := tunnel.SessionID(request.GetSession().GetSessionId()) clientInfo := s.state.GetClient(sessionID) @@ -198,7 +195,7 @@ func (s *service) GetTelepresenceAPI(ctx context.Context, e *empty.Empty) (*rpc. // ArriveAsClient establishes a session between a client and the Manager. func (s *service) ArriveAsClient(ctx context.Context, client *rpc.ClientInfo) (*rpc.SessionInfo, error) { - dlog.Debugf(ctx, "ArriveAsClient called, namespace: %s", client.Namespace) + dlog.Debugf(ctx, "Namespace: %s", client.Namespace) if !s.State().ManagesNamespace(ctx, client.Namespace) { return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("namespace %s is not managed", client.Namespace)) @@ -222,7 +219,7 @@ func (s *service) ArriveAsClient(ctx context.Context, client *rpc.ClientInfo) (* // ArriveAsAgent establishes a session between an agent and the Manager. func (s *service) ArriveAsAgent(ctx context.Context, agent *rpc.AgentInfo) (*rpc.SessionInfo, error) { - dlog.Debugf(ctx, "ArriveAsAgent %s(%s) called", agent.PodName, agent.PodIp) + dlog.Debugf(ctx, "Name %s, IP %s", agent.PodName, agent.PodIp) if val := validateAgent(agent); val != "" { return nil, status.Error(codes.InvalidArgument, val) } @@ -248,8 +245,6 @@ func (s *service) ReportMetrics(ctx context.Context, metrics *rpc.TunnelMetrics) } func (s *service) GetClientConfig(ctx context.Context, _ *empty.Empty) (*rpc.CLIConfig, error) { - dlog.Debug(ctx, "GetClientConfig called") - return &rpc.CLIConfig{ ConfigYaml: s.configWatcher.GetClientConfigYaml(ctx), }, nil @@ -257,15 +252,11 @@ func (s *service) GetClientConfig(ctx context.Context, _ *empty.Empty) (*rpc.CLI // Remain indicates that the session is still valid. func (s *service) Remain(ctx context.Context, req *rpc.RemainRequest) (*empty.Empty, error) { - // ctx = WithSessionInfo(ctx, req.GetSession()) - // dlog.Debug(ctx, "Remain called") sessionID := tunnel.SessionID(req.GetSession().GetSessionId()) if ok := s.state.MarkSession(req, s.clock.Now()); !ok { return nil, status.Errorf(codes.NotFound, "Session %q not found", sessionID) } - s.state.RefreshSessionConsumptionMetrics(sessionID) - return &empty.Empty{}, nil } @@ -273,9 +264,8 @@ func (s *service) Remain(ctx context.Context, req *rpc.RemainRequest) (*empty.Em func (s *service) Depart(ctx context.Context, session *rpc.SessionInfo) (*empty.Empty, error) { ctx = managerutil.WithSessionInfo(ctx, session) sessionID := tunnel.SessionID(session.GetSessionId()) - dlog.Debug(ctx, "Depart called") - // There's reason for the caller to wait for this removal to complete. + // There's no reason for the caller to wait for this removal to complete. go s.state.RemoveSession(context.WithoutCancel(ctx), sessionID) return &empty.Empty{}, nil } @@ -283,9 +273,6 @@ func (s *service) Depart(ctx context.Context, session *rpc.SessionInfo) (*empty. // WatchAgentPods notifies a client of the set of known Agents. func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_WatchAgentPodsServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), session) - dlog.Debug(ctx, "WatchAgentPods called") - defer dlog.Debug(ctx, "WatchAgentPods ended") - clientSession := tunnel.SessionID(session.SessionId) clientInfo := s.state.GetClient(clientSession) if clientInfo == nil { @@ -363,7 +350,6 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa // WatchAgents notifies a client of the set of known Agents in the connected namespace. func (s *service) WatchAgents(session *rpc.SessionInfo, stream rpc.Manager_WatchAgentsServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), session) - dlog.Debug(ctx, "WatchAgents called") clientInfo := s.state.GetClient(tunnel.SessionID(session.SessionId)) if clientInfo == nil { return status.Errorf(codes.NotFound, "Client session %q not found", session.SessionId) @@ -375,7 +361,6 @@ func (s *service) WatchAgents(session *rpc.SessionInfo, stream rpc.Manager_Watch // WatchAgentsNS notifies a client of the set of known Agents in the namespaces given in the request. func (s *service) WatchAgentsNS(request *rpc.AgentsRequest, stream rpc.Manager_WatchAgentsNSServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), request.Session) - dlog.Debug(ctx, "WatchAgentsNS called") return s.watchAgents(ctx, func(_ tunnel.SessionID, a *state.AgentSession) bool { return slices.Contains(request.Namespaces, a.Namespace) }, stream) @@ -469,9 +454,6 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(tunnel.Sess func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_WatchInterceptsServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), session) sessionID := tunnel.SessionID(session.GetSessionId()) - - dlog.Debug(ctx, "WatchIntercepts called") - var sessionDone <-chan struct{} var filter func(id string, info *state.Intercept) bool if sessionID == "" { @@ -559,7 +541,7 @@ func (s *service) PrepareIntercept(ctx context.Context, request *rpc.CreateInter } }() ctx = managerutil.WithSessionInfo(ctx, request.Session) - dlog.Debugf(ctx, "PrepareIntercept %s called", request.InterceptSpec.Name) + dlog.Debugf(ctx, "Intercept name %s", request.InterceptSpec.Name) return s.state.PrepareIntercept(ctx, request) } @@ -568,7 +550,6 @@ func (s *service) GetKnownWorkloadKinds(ctx context.Context, request *rpc.Sessio return nil, err } ctx = managerutil.WithSessionInfo(ctx, request) - dlog.Debugf(ctx, "GetKnownWorkloadKinds called") enabledWorkloadKinds := managerutil.GetEnv(ctx).EnabledWorkloadKinds kinds := make([]rpc.WorkloadInfo_Kind, len(enabledWorkloadKinds)) for i, wlKind := range enabledWorkloadKinds { @@ -580,7 +561,6 @@ func (s *service) GetKnownWorkloadKinds(ctx context.Context, request *rpc.Sessio func (s *service) EnsureAgent(ctx context.Context, request *rpc.EnsureAgentRequest) (*rpc.AgentInfoSnapshot, error) { session := request.GetSession() ctx = managerutil.WithSessionInfo(ctx, session) - dlog.Debugf(ctx, "EnsureAgent called") sessionID := tunnel.SessionID(session.GetSessionId()) client := s.state.GetClient(sessionID) if client == nil { @@ -604,7 +584,7 @@ func (s *service) EnsureAgent(ctx context.Context, request *rpc.EnsureAgentReque func (s *service) CreateIntercept(ctx context.Context, ciReq *rpc.CreateInterceptRequest) (*rpc.InterceptInfo, error) { ctx = managerutil.WithSessionInfo(ctx, ciReq.GetSession()) spec := ciReq.InterceptSpec - dlog.Debugf(ctx, "CreateIntercept %s called", ciReq.InterceptSpec.Name) + dlog.Debugf(ctx, "Intercept name %s", ciReq.InterceptSpec.Name) if val := validateIntercept(spec); val != "" { return nil, status.Error(codes.InvalidArgument, val) @@ -650,7 +630,7 @@ func (s *service) RemoveIntercept(ctx context.Context, riReq *rpc.RemoveIntercep sessionID := tunnel.SessionID(riReq.GetSession().GetSessionId()) name := riReq.Name - dlog.Debugf(ctx, "RemoveIntercept called: %s", name) + dlog.Debugf(ctx, "Intercept name %s", name) client := s.state.GetClient(sessionID) if client == nil { @@ -688,9 +668,9 @@ func (s *service) ReviewIntercept(ctx context.Context, rIReq *rpc.ReviewIntercep } if rIReq.Disposition == rpc.InterceptDispositionType_AGENT_ERROR { - dlog.Errorf(ctx, "ReviewIntercept called: %s - %s: %s", ceptID, rIReq.Disposition, rIReq.Message) + dlog.Errorf(ctx, "%s - %s: %s", ceptID, rIReq.Disposition, rIReq.Message) } else { - dlog.Debugf(ctx, "ReviewIntercept called: %s - %s", ceptID, rIReq.Disposition) + dlog.Debugf(ctx, "%s - %s", ceptID, rIReq.Disposition) } s.removeExcludedEnvVars(rIReq.Environment) @@ -746,16 +726,12 @@ func (s *service) Tunnel(server rpc.Manager_TunnelServer) error { func (s *service) WatchDial(session *rpc.SessionInfo, stream rpc.Manager_WatchDialServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), session) - dlog.Debugf(ctx, "WatchDial called") lrCh := s.state.WatchDial(tunnel.SessionID(session.SessionId)) for { select { // connection broken case <-ctx.Done(): return nil - // service stopped - case <-s.ctx.Done(): - return nil case lr := <-lrCh: if lr == nil { return nil @@ -945,18 +921,17 @@ func (s *service) LookupDNS(ctx context.Context, request *rpc.DNSRequest) (respo func (s *service) AgentLookupDNSResponse(ctx context.Context, response *rpc.DNSAgentResponse) (*empty.Empty, error) { ctx = managerutil.WithSessionInfo(ctx, response.GetSession()) - dlog.Debugf(ctx, "AgentLookupDNSResponse called %s", response.Request.Name) + dlog.Debugf(ctx, "name: %s", response.Request.Name) s.state.PostLookupDNSResponse(ctx, response) return &empty.Empty{}, nil } func (s *service) WatchLookupDNS(session *rpc.SessionInfo, stream rpc.Manager_WatchLookupDNSServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), session) - dlog.Debugf(ctx, "WatchLookupDNS called") rqCh := s.state.WatchLookupDNS(tunnel.SessionID(session.SessionId)) for { select { - case <-s.ctx.Done(): + case <-ctx.Done(): return nil case rq := <-rqCh: if rq == nil { @@ -988,18 +963,16 @@ func (s *service) SetLogLevel(ctx context.Context, request *rpc.LogLevelRequest) func (s *service) UninstallAgents(ctx context.Context, request *rpc.UninstallAgentsRequest) (*empty.Empty, error) { ctx = managerutil.WithSessionInfo(ctx, request.GetSessionInfo()) - dlog.Debugf(ctx, "UninstallAgents called %s", request.Agents) + dlog.Debugf(ctx, "%s", request.Agents) return &empty.Empty{}, s.state.UninstallAgents(ctx, request) } func (s *service) WatchLogLevel(_ *empty.Empty, stream rpc.Manager_WatchLogLevelServer) error { - dlog.Debugf(stream.Context(), "WatchLogLevel called") return s.state.WaitForTempLogLevel(stream) } func (s *service) WatchClusterInfo(session *rpc.SessionInfo, stream rpc.Manager_WatchClusterInfoServer) error { ctx := managerutil.WithSessionInfo(stream.Context(), session) - dlog.Debugf(ctx, "WatchClusterInfo called") return s.clusterInfo.Watch(ctx, stream) } @@ -1016,9 +989,8 @@ func (s *service) WatchWorkloads(request *rpc.WorkloadEventsRequest, stream rpc. dlog.Errorf(ctx, "WatchWorkloads panic: %+v", err) err = status.Error(codes.Internal, err.Error()) } - dlog.Debugf(ctx, "WatchWorkloads ended") }() - dlog.Debugf(ctx, "WatchWorkloads called, namespace %q", request.Namespace) + dlog.Debugf(ctx, "Namespace %q", request.Namespace) if request.SessionInfo == nil { return status.Error(codes.InvalidArgument, "SessionInfo is required") diff --git a/cmd/traffic/cmd/manager/state/state.go b/cmd/traffic/cmd/manager/state/state.go index 9109a9a4ba..30a7c6ea10 100644 --- a/cmd/traffic/cmd/manager/state/state.go +++ b/cmd/traffic/cmd/manager/state/state.go @@ -198,17 +198,14 @@ func NewState(ctx context.Context) State { // pruneSessions will remove all sessions that belong to namespaces that are no longer managed. func (s *state) pruneSessions(ctx context.Context) { nss := namespaces.Get(ctx) - var sids []tunnel.SessionID - s.clients.Range(func(id tunnel.SessionID, c *ClientSession) bool { - if !slices.Contains(nss, c.Namespace) { - sids = append(sids, id) + s.clients.Range(func(id tunnel.SessionID, cs *ClientSession) bool { + if !slices.Contains(nss, cs.Namespace) { + s.clients.Delete(id) + s.removeClientSession(ctx, cs) } return true }) - for _, sid := range sids { - s.removeClientSession(ctx, sid) - } - clear(sids) + var sids []tunnel.SessionID s.agents.Range(func(s tunnel.SessionID, c *AgentSession) bool { if !slices.Contains(nss, c.Namespace) { sids = append(sids, s) @@ -306,8 +303,8 @@ func (s *state) MarkSession(req *rpc.RemainRequest, now time.Time) (ok bool) { // RemoveSession removes an AgentSession from the set of present session IDs. func (s *state) RemoveSession(ctx context.Context, id tunnel.SessionID) { - if _, ok := s.clients.Load(id); ok { - s.removeClientSession(ctx, id) + if cs, ok := s.clients.LoadAndDelete(id); ok { + s.removeClientSession(ctx, cs) } else { s.removeAgentSession(ctx, id) } @@ -315,41 +312,28 @@ func (s *state) RemoveSession(ctx context.Context, id tunnel.SessionID) { // removeAgentSession removes an AgentSession from the set of present session IDs. func (s *state) removeAgentSession(ctx context.Context, id tunnel.SessionID) { - s.agents.Compute(id, func(agent *AgentSession, loaded bool) (*AgentSession, bool) { - if !loaded { - return nil, true - } + if as, loaded := s.agents.LoadAndDelete(id); loaded { dlog.Debugf(ctx, "AgentSession %s removed. Explicit removal", id) - - // kill the session - defer agent.Cancel() - s.consolidateAgentSessionIntercepts(ctx, agent) - return nil, true - }) + mutator.GetMap(s.backgroundCtx).Inactivate(types.UID(as.PodUid)) + s.consolidateAgentSessionIntercepts(ctx, as) + } } // removeClientSession removes an AgentSession from the set of present session IDs. -func (s *state) removeClientSession(ctx context.Context, id tunnel.SessionID) { - s.clients.Compute(id, func(client *ClientSession, loaded bool) (*ClientSession, bool) { - if !loaded { - return nil, true - } - dlog.Debugf(ctx, "ClientSession %s removed. Explicit removal", id) - - // kill the session - defer client.Cancel() - s.gcClientSessionIntercepts(ctx, id, client) - scm := client.consumptionMetrics - atomic.AddUint64(&s.tunnelIngressCounter, scm.FromClientBytes.GetValue()) - atomic.AddUint64(&s.tunnelEgressCounter, scm.ToClientBytes.GetValue()) - s.allClientSessionsFinalizerCall(client) - return nil, true - }) +func (s *state) removeClientSession(ctx context.Context, cs *ClientSession) { + dlog.Debugf(ctx, "ClientSession %s removed. Explicit removal", cs.ID()) + + // kill the session + cs.Cancel() + s.gcClientSessionIntercepts(ctx, cs) + scm := cs.consumptionMetrics + atomic.AddUint64(&s.tunnelIngressCounter, scm.FromClientBytes.GetValue()) + atomic.AddUint64(&s.tunnelEgressCounter, scm.ToClientBytes.GetValue()) + s.allClientSessionsFinalizerCall(cs) } func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *AgentSession) { dlog.Debugf(ctx, "Consolidating intercepts after removal of agent %s(%s)", agent.PodName, agent.PodIp) - mutator.GetMap(s.backgroundCtx).Inactivate(types.UID(agent.PodUid)) s.intercepts.Range(func(interceptID string, intercept *Intercept) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED || agent.PodIp != intercept.PodIp { // Not of interest. Continue iteration. @@ -363,7 +347,7 @@ func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *Ag intercept.Disposition = errCode intercept.Message = errMsg }) - } else { + } else if agent.PodIp == intercept.PodIp { // The agent is about to die, but apparently more agents are present. Let some other agent pick it up then. dlog.Debugf(ctx, "Intercept %q lost its agent pod %s(%s). Setting it disposition to WAITING", interceptID, agent.PodName, agent.PodIp) s.UpdateIntercept(interceptID, func(intercept *Intercept) { @@ -372,18 +356,17 @@ func (s *state) consolidateAgentSessionIntercepts(ctx context.Context, agent *Ag intercept.Disposition = rpc.InterceptDispositionType_WAITING }) } - // Only one agent with this IP. Break iteration. - return false + return true }) } -func (s *state) gcClientSessionIntercepts(ctx context.Context, id tunnel.SessionID, client *ClientSession) { +func (s *state) gcClientSessionIntercepts(ctx context.Context, client *ClientSession) { // GC all intercepts for the client session (intercept.ClientSession.SessionId) s.intercepts.Range(func(interceptID string, intercept *Intercept) bool { if intercept.Disposition == rpc.InterceptDispositionType_REMOVED { return true } - if tunnel.SessionID(intercept.ClientSession.SessionId) == id { + if tunnel.SessionID(intercept.ClientSession.SessionId) == client.ID() { // Client went away: // Delete it. wl := strings.SplitN(interceptID, ":", 2)[1] @@ -400,7 +383,8 @@ func (s *state) ExpireSessions(ctx context.Context, clientMoment, agentMoment ti s.clients.Range(func(id tunnel.SessionID, client *ClientSession) bool { moment := clientMoment if client.LastMarked().Before(moment) { - s.removeClientSession(ctx, id) + s.clients.Delete(id) + s.removeClientSession(ctx, client) } return true }) diff --git a/go.mod b/go.mod index 271adc2992..75f95d7191 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/golang/mock v1.7.0-rc.1 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0 github.com/hashicorp/go-multierror v1.1.1 github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb github.com/miekg/dns v1.1.63 diff --git a/go.sum b/go.sum index f2152232e3..a78554aeb9 100644 --- a/go.sum +++ b/go.sum @@ -201,6 +201,8 @@ github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0 h1:kQ0NI7W1B3HwiN5gAYtY+XFItDPbLBwYRxAqbFTyDes= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0/go.mod h1:zrT2dxOAjNFPRGjTUe2Xmb4q4YdUwVvQFV6xiCSf+z0= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= diff --git a/integration_test/argo_rollouts_test.go b/integration_test/argo_rollouts_test.go index 8dc5993d86..20fa9ae39c 100644 --- a/integration_test/argo_rollouts_test.go +++ b/integration_test/argo_rollouts_test.go @@ -105,14 +105,7 @@ func (s *argoRolloutsSuite) Test_SuccessfullyInterceptsArgoRollout() { itest.TelepresenceOk(ctx, "leave", svc) stdout = itest.TelepresenceOk(ctx, "list", "--intercepts") require.NotContains(stdout, svc+": intercepted") - - itest.TelepresenceDisconnectOk(ctx) - - dfltCtx := itest.WithUser(ctx, "default") - itest.TelepresenceOk(dfltCtx, "connect", "--namespace", s.AppNamespace(), "--manager-namespace", s.ManagerNamespace()) - itest.TelepresenceOk(dfltCtx, "uninstall", svc) - itest.TelepresenceDisconnectOk(dfltCtx) - s.TelepresenceConnect(ctx) + itest.TelepresenceOk(ctx, "uninstall", svc) require.Eventually( func() bool { diff --git a/integration_test/podscaling_test.go b/integration_test/podscaling_test.go index 528a13369e..f6f164d1e4 100644 --- a/integration_test/podscaling_test.go +++ b/integration_test/podscaling_test.go @@ -25,6 +25,12 @@ func (s *interceptMountSuite) Test_RestartInterceptedPod() { // Scale down to zero pods require.NoError(s.Kubectl(ctx, "scale", "deploy", s.ServiceName(), "--replicas", "0")) + scaleUp := true + defer func() { + if scaleUp { + assert.NoError(s.Kubectl(ctx, "scale", "deploy", s.ServiceName(), "--replicas", "1")) + } + }() // Wait until the pods have terminated. This might take a long time (several minutes). require.Eventually(func() bool { return len(s.runningPods(ctx)) == 0 }, 2*time.Minute, 6*time.Second) @@ -49,6 +55,8 @@ func (s *interceptMountSuite) Test_RestartInterceptedPod() { // Scale up again (start intercepted pod) assert.NoError(s.Kubectl(ctx, "scale", "deploy", s.ServiceName(), "--replicas", "1")) + scaleUp = false + assert.Eventually(func() bool { return len(s.runningPods(ctx)) == 1 }, itest.PodCreateTimeout(ctx), 6*time.Second) s.CapturePodLogs(ctx, s.ServiceName(), "traffic-agent", s.AppNamespace()) diff --git a/pkg/agentmap/discorvery.go b/pkg/agentmap/discorvery.go index 494324fa0a..b9ade3a115 100644 --- a/pkg/agentmap/discorvery.go +++ b/pkg/agentmap/discorvery.go @@ -23,6 +23,12 @@ import ( var ReplicaSetNameRx = regexp.MustCompile(`\A(.+)-[a-f0-9]+\z`) +type WorkloadNotFoundError string + +func (e WorkloadNotFoundError) Error() string { + return string(e) +} + func FindOwnerWorkload(ctx context.Context, obj k8sapi.Object, supportedWorkloadKinds k8sapi.Kinds) (k8sapi.Workload, error) { dlog.Tracef(ctx, "FindOwnerWorkload(%s,%s,%s)", obj.GetName(), obj.GetNamespace(), obj.GetKind()) lbs := obj.GetLabels() @@ -63,7 +69,7 @@ func FindOwnerWorkload(ctx context.Context, obj k8sapi.Object, supportedWorkload if wl, ok := obj.(k8sapi.Workload); ok { return wl, nil } - return nil, fmt.Errorf("unable to find workload owner for %s.%s", obj.GetName(), obj.GetNamespace()) + return nil, WorkloadNotFoundError(fmt.Sprintf("unable to find workload owner for %s.%s", obj.GetName(), obj.GetNamespace())) } func GetWorkload(ctx context.Context, name, namespace string, workloadKind k8sapi.Kind) (obj k8sapi.Workload, err error) { diff --git a/pkg/client/userd/daemon/grpc.go b/pkg/client/userd/daemon/grpc.go index cb728ac835..33801095c8 100644 --- a/pkg/client/userd/daemon/grpc.go +++ b/pkg/client/userd/daemon/grpc.go @@ -18,7 +18,6 @@ import ( "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dexec" - "github.com/datawire/dlib/dgroup" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/rpc/v2/common" rpc "github.com/telepresenceio/telepresence/rpc/v2/connector" @@ -41,60 +40,25 @@ func callRecovery(c context.Context, r any, err error) error { return err } -type reqNumberKey struct{} - -func getReqNumber(ctx context.Context) int64 { - num := ctx.Value(reqNumberKey{}) - if num == nil { - return 0 - } - return num.(int64) -} - -func withReqNumber(ctx context.Context, num int64) context.Context { - return context.WithValue(ctx, reqNumberKey{}, num) -} - -func (s *service) callCtx(ctx context.Context, name string) context.Context { - num := atomic.AddInt64(&s.ucn, 1) - ctx = withReqNumber(ctx, num) - return dgroup.WithGoroutineName(ctx, fmt.Sprintf("/%s-%d", name, num)) -} - -func (s *service) LogCall(c context.Context, callName string, f func(context.Context)) { - c = s.callCtx(c, callName) - dlog.Debug(c, "called") - defer dlog.Debug(c, "returned") - f(c) -} - func (s *service) FuseFTPError() error { return s.fuseFTPError } -func (s *service) WithSession(c context.Context, callName string, f func(context.Context, userd.Session) error) (err error) { - s.LogCall(c, callName, func(_ context.Context) { - if atomic.LoadInt32(&s.sessionQuitting) != 0 { - err = status.Error(codes.Canceled, "session cancelled") - return - } - s.sessionLock.RLock() - defer s.sessionLock.RUnlock() - if s.session == nil { - err = status.Error(codes.Unavailable, "no active session") - return - } - if s.sessionContext.Err() != nil { - // Session context has been cancelled - err = status.Error(codes.Canceled, "session cancelled") - return - } - defer func() { err = callRecovery(c, recover(), err) }() - num := getReqNumber(c) - ctx := dgroup.WithGoroutineName(s.sessionContext, fmt.Sprintf("/%s-%d", callName, num)) - err = f(ctx, s.session) - }) - return +func (s *service) WithSession(c context.Context, f func(context.Context, userd.Session) error) (err error) { + if atomic.LoadInt32(&s.sessionQuitting) != 0 { + return status.Error(codes.Canceled, "session cancelled") + } + s.sessionLock.RLock() + defer s.sessionLock.RUnlock() + if s.session == nil { + return status.Error(codes.Unavailable, "no active session") + } + if s.sessionContext.Err() != nil { + // Session context has been cancelled + return status.Error(codes.Canceled, "session cancelled") + } + defer func() { err = callRecovery(c, recover(), err) }() + return f(s.sessionContext, s.session) } func (s *service) Version(_ context.Context, _ *empty.Empty) (*common.VersionInfo, error) { @@ -119,39 +83,33 @@ func (c crImpl) Request() *rpc.ConnectRequest { } func (s *service) Connect(ctx context.Context, cr *rpc.ConnectRequest) (result *rpc.ConnectInfo, err error) { - s.LogCall(ctx, "Connect", func(c context.Context) { - if err = s.PostConnectRequest(ctx, crImpl{ConnectRequest: cr}); err == nil { - result, err = s.ReadConnectResponse(ctx) - } - }) + if err = s.PostConnectRequest(ctx, crImpl{ConnectRequest: cr}); err == nil { + result, err = s.ReadConnectResponse(ctx) + } return result, err } func (s *service) Disconnect(ctx context.Context, ex *empty.Empty) (*empty.Empty, error) { - s.LogCall(ctx, "Disconnect", func(ctx context.Context) { - s.cancelSession() - _ = s.withRootDaemon(ctx, func(ctx context.Context, rd daemon.DaemonClient) error { - _, err := rd.Disconnect(ctx, ex) - return err - }) + s.cancelSession() + _ = s.withRootDaemon(ctx, func(ctx context.Context, rd daemon.DaemonClient) error { + _, err := rd.Disconnect(ctx, ex) + return err }) return &empty.Empty{}, nil } func (s *service) Status(ctx context.Context, ex *empty.Empty) (result *rpc.ConnectInfo, err error) { - s.LogCall(ctx, "Status", func(c context.Context) { - s.sessionLock.RLock() - defer s.sessionLock.RUnlock() - if s.session == nil { - result = &rpc.ConnectInfo{Error: rpc.ConnectInfo_DISCONNECTED} - _ = s.withRootDaemon(c, func(c context.Context, dc daemon.DaemonClient) error { - result.DaemonStatus, err = dc.Status(c, ex) - return nil - }) - } else { - result = s.session.Status(s.sessionContext) - } - }) + s.sessionLock.RLock() + defer s.sessionLock.RUnlock() + if s.session == nil { + result = &rpc.ConnectInfo{Error: rpc.ConnectInfo_DISCONNECTED} + _ = s.withRootDaemon(ctx, func(c context.Context, dc daemon.DaemonClient) error { + result.DaemonStatus, err = dc.Status(c, ex) + return nil + }) + } else { + result = s.session.Status(s.sessionContext) + } return } @@ -233,7 +191,7 @@ func (s *service) CanIntercept(c context.Context, ir *rpc.CreateInterceptRequest } scout.Report(c, action, entries...) }() - err = s.WithSession(c, "CanIntercept", func(c context.Context, session userd.Session) error { + err = s.WithSession(c, func(c context.Context, session userd.Session) error { _, result = session.CanIntercept(c, ir) if result == nil { result = &rpc.InterceptResult{Error: common.InterceptError_UNSPECIFIED} @@ -256,7 +214,7 @@ func (s *service) CreateIntercept(c context.Context, ir *rpc.CreateInterceptRequ } scout.Report(c, action, entries...) }() - err = s.WithSession(c, "CreateIntercept", func(c context.Context, session userd.Session) error { + err = s.WithSession(c, func(c context.Context, session userd.Session) error { result = session.AddIntercept(c, ir) entries, ok = s.scoutInterceptEntries(c, ir.GetSpec(), result) return nil @@ -277,7 +235,7 @@ func (s *service) RemoveIntercept(c context.Context, rr *manager.RemoveIntercept } scout.Report(c, action, entries...) }() - err = s.WithSession(c, "RemoveIntercept", func(c context.Context, session userd.Session) error { + err = s.WithSession(c, func(c context.Context, session userd.Session) error { result = &rpc.InterceptResult{} spec = session.GetInterceptSpec(rr.Name) if spec != nil { @@ -302,7 +260,7 @@ func (s *service) RemoveIntercept(c context.Context, rr *manager.RemoveIntercept } func (s *service) UpdateIntercept(c context.Context, rr *manager.UpdateInterceptRequest) (result *manager.InterceptInfo, err error) { - err = s.WithSession(c, "UpdateIntercept", func(c context.Context, session userd.Session) error { + err = s.WithSession(c, func(c context.Context, session userd.Session) error { result, err = session.ManagerClient().UpdateIntercept(c, rr) return err }) @@ -310,19 +268,19 @@ func (s *service) UpdateIntercept(c context.Context, rr *manager.UpdateIntercept } func (s *service) AddInterceptor(ctx context.Context, interceptor *rpc.Interceptor) (*empty.Empty, error) { - return &empty.Empty{}, s.WithSession(ctx, "AddInterceptor", func(_ context.Context, session userd.Session) error { + return &empty.Empty{}, s.WithSession(ctx, func(_ context.Context, session userd.Session) error { return session.AddInterceptor(ctx, interceptor.InterceptId, interceptor) }) } func (s *service) RemoveInterceptor(ctx context.Context, interceptor *rpc.Interceptor) (*empty.Empty, error) { - return &empty.Empty{}, s.WithSession(ctx, "RemoveInterceptor", func(_ context.Context, session userd.Session) error { + return &empty.Empty{}, s.WithSession(ctx, func(_ context.Context, session userd.Session) error { return session.RemoveInterceptor(interceptor.InterceptId) }) } func (s *service) List(c context.Context, lr *rpc.ListRequest) (result *rpc.WorkloadInfoSnapshot, err error) { - err = s.WithSession(c, "List", func(c context.Context, session userd.Session) error { + err = s.WithSession(c, func(c context.Context, session userd.Session) error { result, err = session.WorkloadInfoSnapshot(c, []string{lr.Namespace}, lr.Filter) return err }) @@ -330,7 +288,7 @@ func (s *service) List(c context.Context, lr *rpc.ListRequest) (result *rpc.Work } func (s *service) GetKnownWorkloadKinds(ctx context.Context, _ *empty.Empty) (result *manager.KnownWorkloadKinds, err error) { - err = s.WithSession(ctx, "GetKnownWorkloadKinds", func(ctx context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { result, err = session.ManagerClient().GetKnownWorkloadKinds(ctx, session.SessionInfo()) if err != nil { if status.Code(err) != codes.Unimplemented { @@ -352,7 +310,7 @@ func (s *service) WatchWorkloads(wr *rpc.WatchWorkloadsRequest, stream rpc.Conne var sessionCtx context.Context var session userd.Session - err := s.WithSession(stream.Context(), "WatchWorkloads", func(c context.Context, s userd.Session) error { + err := s.WithSession(stream.Context(), func(c context.Context, s userd.Session) error { session, sessionCtx = s, c return nil }) @@ -364,7 +322,7 @@ func (s *service) WatchWorkloads(wr *rpc.WatchWorkloadsRequest, stream rpc.Conne } func (s *service) Uninstall(c context.Context, ur *rpc.UninstallRequest) (result *common.Result, err error) { - err = s.WithSession(c, "Uninstall", func(c context.Context, session userd.Session) error { + err = s.WithSession(c, func(c context.Context, session userd.Session) error { result, err = session.Uninstall(c, ur) return err }) @@ -372,7 +330,7 @@ func (s *service) Uninstall(c context.Context, ur *rpc.UninstallRequest) (result } func (s *service) GetConfig(ctx context.Context, _ *empty.Empty) (cfg *rpc.ClientConfig, err error) { - err = s.WithSession(ctx, "GetConfig", func(c context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(c context.Context, session userd.Session) error { sc, err := session.GetConfig(ctx) if err != nil { return err @@ -388,7 +346,7 @@ func (s *service) GetConfig(ctx context.Context, _ *empty.Empty) (cfg *rpc.Clien } func (s *service) GatherLogs(ctx context.Context, request *rpc.LogsRequest) (result *rpc.LogsResponse, err error) { - err = s.WithSession(ctx, "GatherLogs", func(c context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(c context.Context, session userd.Session) error { result, err = session.GatherLogs(c, request) return err }) @@ -396,55 +354,51 @@ func (s *service) GatherLogs(ctx context.Context, request *rpc.LogsRequest) (res } func (s *service) SetLogLevel(ctx context.Context, request *rpc.LogLevelRequest) (result *empty.Empty, err error) { - s.LogCall(ctx, "SetLogLevel", func(c context.Context) { - mrq := &manager.LogLevelRequest{ - LogLevel: request.LogLevel, - Duration: request.Duration, - } - setLocal := func() { - duration := time.Duration(0) - if request.Duration != nil { - duration = request.Duration.AsDuration() - } - if err = logging.SetAndStoreTimedLevel(ctx, s.timedLogLevel, request.LogLevel, duration, userd.ProcessName); err != nil { - err = status.Error(codes.Internal, err.Error()) - } else if !s.rootSessionInProc { - err = s.withRootDaemon(ctx, func(ctx context.Context, rd daemon.DaemonClient) error { - _, err := rd.SetLogLevel(ctx, mrq) - return err - }) - } + mrq := &manager.LogLevelRequest{ + LogLevel: request.LogLevel, + Duration: request.Duration, + } + setLocal := func() { + duration := time.Duration(0) + if request.Duration != nil { + duration = request.Duration.AsDuration() } - setRemote := func() { - err = s.WithSession(ctx, "SetLogLevel", func(ctx context.Context, session userd.Session) error { - _, err := session.ManagerClient().SetLogLevel(ctx, mrq) + if err = logging.SetAndStoreTimedLevel(ctx, s.timedLogLevel, request.LogLevel, duration, userd.ProcessName); err != nil { + err = status.Error(codes.Internal, err.Error()) + } else if !s.rootSessionInProc { + err = s.withRootDaemon(ctx, func(ctx context.Context, rd daemon.DaemonClient) error { + _, err := rd.SetLogLevel(ctx, mrq) return err }) } - switch request.Scope { - case rpc.LogLevelRequest_LOCAL_ONLY: - setLocal() - case rpc.LogLevelRequest_REMOTE_ONLY: + } + setRemote := func() { + err = s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { + _, err := session.ManagerClient().SetLogLevel(ctx, mrq) + return err + }) + } + switch request.Scope { + case rpc.LogLevelRequest_LOCAL_ONLY: + setLocal() + case rpc.LogLevelRequest_REMOTE_ONLY: + setRemote() + default: + setLocal() + if err == nil { setRemote() - default: - setLocal() - if err == nil { - setRemote() - } } - }) + } return &empty.Empty{}, err } func (s *service) Quit(ctx context.Context, ex *empty.Empty) (*empty.Empty, error) { - s.LogCall(ctx, "Quit", func(c context.Context) { - s.cancelSession() - s.quit() - _ = s.withRootDaemon(context.WithoutCancel(ctx), func(ctx context.Context, rd daemon.DaemonClient) error { - dlog.Debug(ctx, "Telling root daemon to Quit") - _, err := rd.Quit(ctx, ex) - return err - }) + s.cancelSession() + s.quit() + _ = s.withRootDaemon(context.WithoutCancel(ctx), func(ctx context.Context, rd daemon.DaemonClient) error { + dlog.Debug(ctx, "Telling root daemon to Quit") + _, err := rd.Quit(ctx, ex) + return err }) return ex, nil } @@ -486,7 +440,7 @@ func (s *service) RemoteMountAvailability(ctx context.Context, _ *empty.Empty) ( func (s *service) GetNamespaces(ctx context.Context, req *rpc.GetNamespacesRequest) (*rpc.GetNamespacesResponse, error) { var resp rpc.GetNamespacesResponse - err := s.WithSession(ctx, "GetNamespaces", func(ctx context.Context, session userd.Session) error { + err := s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { resp.Namespaces = session.GetCurrentNamespaces(req.ForClientAccess) return nil }) @@ -508,7 +462,7 @@ func (s *service) GetNamespaces(ctx context.Context, req *rpc.GetNamespacesReque } func (s *service) TrafficManagerVersion(ctx context.Context, _ *empty.Empty) (vi *common.VersionInfo, err error) { - err = s.WithSession(ctx, "TrafficManagerVersion", func(ctx context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { vi = &common.VersionInfo{Name: session.ManagerName(), Version: "v" + session.ManagerVersion().String()} return nil }) @@ -524,7 +478,7 @@ func (s *service) RootDaemonVersion(ctx context.Context, empty *empty.Empty) (vi } func (s *service) AgentImageFQN(ctx context.Context, empty *emptypb.Empty) (fqn *manager.AgentImageFQN, err error) { - err = s.WithSession(ctx, "AgentImageFQN", func(ctx context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { fqn, err = session.ManagerClient().GetAgentImageFQN(ctx, empty) return err }) @@ -532,7 +486,7 @@ func (s *service) AgentImageFQN(ctx context.Context, empty *emptypb.Empty) (fqn } func (s *service) GetAgentConfig(ctx context.Context, request *manager.AgentConfigRequest) (rsp *manager.AgentConfigResponse, err error) { - err = s.WithSession(ctx, "AgentConfig", func(ctx context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { request.Session = session.SessionInfo() rsp, err = session.ManagerClient().GetAgentConfig(ctx, request) return err @@ -543,7 +497,7 @@ func (s *service) GetAgentConfig(ctx context.Context, request *manager.AgentConf func (s *service) GetClusterSubnets(ctx context.Context, _ *empty.Empty) (cs *rpc.ClusterSubnets, err error) { podSubnets := []*manager.IPNet{} svcSubnets := []*manager.IPNet{} - err = s.WithSession(ctx, "GetClusterSubnets", func(ctx context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { // The manager can sometimes send the different subnets in different Sends, // but after 5 seconds of listening to it, we should expect to have everything tCtx, tCancel := context.WithTimeout(ctx, 5*time.Second) @@ -573,7 +527,7 @@ func (s *service) GetClusterSubnets(ctx context.Context, _ *empty.Empty) (cs *rp } func (s *service) GetIntercept(ctx context.Context, request *manager.GetInterceptRequest) (ii *manager.InterceptInfo, err error) { - err = s.WithSession(ctx, "GetIntercept", func(ctx context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { ii = session.GetInterceptInfo(request.Name) if ii == nil { return status.Errorf(codes.NotFound, "found no intercept named %s", request.Name) @@ -584,7 +538,7 @@ func (s *service) GetIntercept(ctx context.Context, request *manager.GetIntercep } func (s *service) SetDNSExcludes(ctx context.Context, req *daemon.SetDNSExcludesRequest) (*emptypb.Empty, error) { - err := s.WithSession(ctx, "SetDNSExcludes", func(ctx context.Context, session userd.Session) error { + err := s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { _, err := session.RootDaemon().SetDNSExcludes(ctx, req) return err }) @@ -592,7 +546,7 @@ func (s *service) SetDNSExcludes(ctx context.Context, req *daemon.SetDNSExcludes } func (s *service) SetDNSMappings(ctx context.Context, req *daemon.SetDNSMappingsRequest) (*emptypb.Empty, error) { - err := s.WithSession(ctx, "SetDNSMappings", func(ctx context.Context, session userd.Session) error { + err := s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { _, err := session.RootDaemon().SetDNSMappings(ctx, req) return err }) @@ -600,7 +554,7 @@ func (s *service) SetDNSMappings(ctx context.Context, req *daemon.SetDNSMappings } func (s *service) Ingest(ctx context.Context, request *rpc.IngestRequest) (response *rpc.IngestInfo, err error) { - err = s.WithSession(ctx, "Ingest", func(ctx context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { response, err = session.Ingest(ctx, request) return err }) @@ -608,7 +562,7 @@ func (s *service) Ingest(ctx context.Context, request *rpc.IngestRequest) (respo } func (s *service) GetIngest(ctx context.Context, request *rpc.IngestIdentifier) (response *rpc.IngestInfo, err error) { - err = s.WithSession(ctx, "GetIngest", func(ctx context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { response, err = session.GetIngest(request) return err }) @@ -616,7 +570,7 @@ func (s *service) GetIngest(ctx context.Context, request *rpc.IngestIdentifier) } func (s *service) LeaveIngest(ctx context.Context, request *rpc.IngestIdentifier) (response *rpc.IngestInfo, err error) { - err = s.WithSession(ctx, "LeaveIngest", func(ctx context.Context, session userd.Session) error { + err = s.WithSession(ctx, func(ctx context.Context, session userd.Session) error { response, err = session.LeaveIngest(ctx, request) return err }) diff --git a/pkg/client/userd/daemon/service.go b/pkg/client/userd/daemon/service.go index a23a919080..bcfa013dcb 100644 --- a/pkg/client/userd/daemon/service.go +++ b/pkg/client/userd/daemon/service.go @@ -59,7 +59,6 @@ type service struct { srv *grpc.Server managerProxy *mgrProxy timedLogLevel log.TimedLevel - ucn int64 fuseFTPError error // The quit function that quits the server. diff --git a/pkg/client/userd/service.go b/pkg/client/userd/service.go index 046a8e3e8a..e1ed4517a2 100644 --- a/pkg/client/userd/service.go +++ b/pkg/client/userd/service.go @@ -21,8 +21,6 @@ type Service interface { // that to the pointer. It will panic if type is not implemented. As(ptr any) - LogCall(context.Context, string, func(context.Context)) - // ListenerAddress returns the address that this service is listening to. ListenerAddress(ctx context.Context) string @@ -36,7 +34,7 @@ type Service interface { FuseFTPMgr() remotefs.FuseFTPManager RootSessionInProcess() bool - WithSession(context.Context, string, func(context.Context, Session) error) error + WithSession(context.Context, func(context.Context, Session) error) error PostConnectRequest(context.Context, ConnectRequest) error ReadConnectResponse(context.Context) (*rpc.ConnectInfo, error) diff --git a/pkg/grpc/server/logging.go b/pkg/grpc/server/logging.go new file mode 100644 index 0000000000..f26ca18a3f --- /dev/null +++ b/pkg/grpc/server/logging.go @@ -0,0 +1,60 @@ +package server + +import ( + "context" + "fmt" + "strings" + "sync/atomic" + + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + + "github.com/datawire/dlib/dgroup" + "github.com/datawire/dlib/dlog" +) + +func interceptorLogger() logging.Logger { + return logging.LoggerFunc(func(ctx context.Context, lvl logging.Level, msg string, fields ...any) { + i := logging.Fields(fields).Iterator() + for i.Next() { + k, v := i.At() + switch k { + case logging.MethodFieldKey: + // Don't log the remain ping unless we're tracing + if v == "Remain" && dlog.MaxLogLevel(ctx) < dlog.LogLevelTrace { + return + } + case + logging.ComponentFieldKey, + logging.ServiceFieldKey, + logging.MethodTypeFieldKey, + "grpc.request.deadline", + "grpc.start_time", + "grpc.time_ms", + "peer.address", + "protocol": + default: + ctx = dlog.WithField(ctx, k, v) + } + } + switch lvl { + case logging.LevelDebug: + // We treat debug logging from GRPC as Trace + dlog.Trace(ctx, msg) + case logging.LevelInfo: + // We treat info logging from GRPC as Debug + dlog.Debug(ctx, msg) + case logging.LevelWarn: + dlog.Warn(ctx, msg) + case logging.LevelError: + dlog.Error(ctx, msg) + } + }) +} + +func callCtx(ctx context.Context, name string, requestCount *uint64) context.Context { + if ix := strings.LastIndexByte(name, '/'); ix >= 0 { + name = name[ix+1:] + } + num := atomic.AddUint64(requestCount, 1) + return dgroup.WithGoroutineName(ctx, fmt.Sprintf("/%s-%d", name, num)) +} diff --git a/pkg/grpc/server/server.go b/pkg/grpc/server/server.go index 994df9adfd..83e8a9c00a 100644 --- a/pkg/grpc/server/server.go +++ b/pkg/grpc/server/server.go @@ -3,7 +3,10 @@ package server import ( "context" "net" + "runtime" + "time" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" "google.golang.org/grpc" "github.com/datawire/dlib/dcontext" @@ -34,16 +37,35 @@ func (s *mergedStream) Context() context.Context { // New creates a gRPC server which has no service registered and has not started to accept requests yet. Values // in the provided context will be included in the context passed to both unary and stream calls. func New(valCtx context.Context, options ...grpc.ServerOption) *grpc.Server { + requestCount := uint64(0) unaryInterceptor := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { - return handler(&mergedCtx{Context: ctx, valCtx: valCtx}, req) + return handler(&mergedCtx{Context: ctx, valCtx: callCtx(valCtx, info.FullMethod, &requestCount)}, req) } streamInterceptor := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { return handler(srv, &mergedStream{ ServerStream: ss, - valCtx: valCtx, + valCtx: callCtx(valCtx, info.FullMethod, &requestCount), }) } - options = append(options, grpc.UnaryInterceptor(unaryInterceptor), grpc.StreamInterceptor(streamInterceptor)) + if dlog.MaxLogLevel(valCtx) >= dlog.LogLevelDebug { + opts := []logging.Option{ + logging.WithLogOnEvents(logging.StartCall, logging.FinishCall), + // Add any other option (check functions starting with logging.With). + } + options = append( + options, + grpc.ChainUnaryInterceptor( + unaryInterceptor, + logging.UnaryServerInterceptor(interceptorLogger(), opts...), + ), + grpc.ChainStreamInterceptor( + streamInterceptor, + logging.StreamServerInterceptor(interceptorLogger(), opts...), + ), + ) + } else { + options = append(options, grpc.UnaryInterceptor(unaryInterceptor), grpc.StreamInterceptor(streamInterceptor)) + } return grpc.NewServer(options...) } @@ -66,6 +88,29 @@ func Serve(ctx context.Context, svc *grpc.Server, lis net.Listener) error { return err } +// Stop the service by calling svc.Stop and give it maxTime to complete before returning. +// If the maxTime expires and the current debug loglevel >= debug, then a stack trace of all goroutines will +// be logged. +func Stop(ctx context.Context, svc *grpc.Server, maxTime time.Duration) { + dead := make(chan struct{}) + dlog.Debug(ctx, "Initiating hard shutdown") + go func() { + defer close(dead) + svc.Stop() + dlog.Debug(ctx, "Hard shutdown complete") + }() + select { + case <-dead: + case <-time.After(maxTime): + // Hard shutdown is stuck! This shouldn't happen, and we need to find out why + if dlog.MaxLogLevel(ctx) >= dlog.LogLevelDebug { + buf := make([]byte, 1024*256) + n := runtime.Stack(buf, true) + dlog.Debug(ctx, string(buf[:n])) + } + } +} + // Wait waits until the given contexts Done channel is closed. The server's GracefulStop function will be called // if the context has soft-cancel enabled. The server's Stop function will be called if no soft-cancel is enabled or // when the GracefulStop doesn't finish until the Done channel of the hard context closed. @@ -84,14 +129,9 @@ func Wait(ctx context.Context, svc *grpc.Server) { case <-dead: // GracefulStop did the job. case <-hardCtx.Done(): - dlog.Debug(ctx, "Initiating hard shutdown") - svc.Stop() - dlog.Debug(ctx, "Hard shutdown complete") + Stop(ctx, svc, 5*time.Second) } } else { - <-ctx.Done() - dlog.Debug(ctx, "Initiating hard shutdown") - svc.Stop() - dlog.Debug(ctx, "Hard shutdown complete") + Stop(ctx, svc, 5*time.Second) } } From ad0fa0d51302d9728cac233c30b1346881b6946e Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 1 Feb 2025 12:26:26 +0100 Subject: [PATCH 38/61] Better error handling in FindOwnerWorkload. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/manager/mutator/watcher.go | 5 ++-- pkg/agentmap/discorvery.go | 9 +++---- pkg/k8sapi/object.go | 16 ++++++++++++ pkg/k8sapi/workload.go | 29 ++++++++++++++++++++++ 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index 7cd0ca3623..a4c4a54f3b 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -2,7 +2,6 @@ package mutator import ( "context" - "errors" "fmt" "slices" "sync" @@ -14,6 +13,7 @@ import ( "google.golang.org/protobuf/types/known/durationpb" core "k8s.io/api/core/v1" v1 "k8s.io/api/policy/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -658,8 +658,7 @@ func podList(ctx context.Context, kind k8sapi.Kind, name, namespace string) ([]* } wl, err := agentmap.FindOwnerWorkload(ctx, k8sapi.Pod(pod), enabledWorkloads) if err != nil { - var ew agentmap.WorkloadNotFoundError - if !errors.As(err, &ew) { + if !k8sErrors.IsNotFound(err) { return nil, err } } else if (kind == "" || wl.GetKind() == kind) && (name == "" || wl.GetName() == name) { diff --git a/pkg/agentmap/discorvery.go b/pkg/agentmap/discorvery.go index b9ade3a115..3ef460a931 100644 --- a/pkg/agentmap/discorvery.go +++ b/pkg/agentmap/discorvery.go @@ -23,10 +23,8 @@ import ( var ReplicaSetNameRx = regexp.MustCompile(`\A(.+)-[a-f0-9]+\z`) -type WorkloadNotFoundError string - -func (e WorkloadNotFoundError) Error() string { - return string(e) +type WorkloadOwnerNotFoundError struct { + *k8sErrors.StatusError } func FindOwnerWorkload(ctx context.Context, obj k8sapi.Object, supportedWorkloadKinds k8sapi.Kinds) (k8sapi.Workload, error) { @@ -69,7 +67,8 @@ func FindOwnerWorkload(ctx context.Context, obj k8sapi.Object, supportedWorkload if wl, ok := obj.(k8sapi.Workload); ok { return wl, nil } - return nil, WorkloadNotFoundError(fmt.Sprintf("unable to find workload owner for %s.%s", obj.GetName(), obj.GetNamespace())) + return nil, &WorkloadOwnerNotFoundError{StatusError: k8sErrors.NewNotFound( + obj.GetGroupResource(), fmt.Sprintf("%s.%s", obj.GetName(), obj.GetNamespace()))} } func GetWorkload(ctx context.Context, name, namespace string, workloadKind k8sapi.Kind) (obj k8sapi.Workload, err error) { diff --git a/pkg/k8sapi/object.go b/pkg/k8sapi/object.go index f5c75e213b..825b4f9759 100644 --- a/pkg/k8sapi/object.go +++ b/pkg/k8sapi/object.go @@ -7,6 +7,7 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" typedCore "k8s.io/client-go/kubernetes/typed/core/v1" ) @@ -21,6 +22,7 @@ type Object interface { Selector() (labels.Selector, error) Update(context.Context) error Patch(context.Context, types.PatchType, []byte, ...string) error + GetGroupResource() schema.GroupResource } func GetService(c context.Context, name, namespace string) (Object, error) { @@ -105,6 +107,13 @@ func (o *service) ki(c context.Context) typedCore.ServiceInterface { return services(c, o.Namespace) } +func (o *service) GetGroupResource() schema.GroupResource { + return schema.GroupResource{ + Group: o.TypeMeta.GroupVersionKind().Group, + Resource: "services", + } +} + func (o *service) GetKind() Kind { return ServiceKind } @@ -156,6 +165,13 @@ func (o *pod) ki(c context.Context) typedCore.PodInterface { return pods(c, o.Namespace) } +func (o *pod) GetGroupResource() schema.GroupResource { + return schema.GroupResource{ + Group: o.TypeMeta.GroupVersionKind().Group, + Resource: "pods", + } +} + func (o *pod) GetKind() Kind { return PodKind } diff --git a/pkg/k8sapi/workload.go b/pkg/k8sapi/workload.go index 97ceeb6ca3..0d8dcf4aa4 100644 --- a/pkg/k8sapi/workload.go +++ b/pkg/k8sapi/workload.go @@ -11,6 +11,7 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" typedApps "k8s.io/client-go/kubernetes/typed/apps/v1" @@ -243,6 +244,13 @@ func (o *deployment) ki(c context.Context) typedApps.DeploymentInterface { return deployments(c, o.Namespace) } +func (o *deployment) GetGroupResource() schema.GroupResource { + return schema.GroupResource{ + Group: o.TypeMeta.GroupVersionKind().Group, + Resource: "deployments", + } +} + func (o *deployment) GetKind() Kind { return DeploymentKind } @@ -312,6 +320,13 @@ func (o *rollout) ki(c context.Context) typedArgoRollouts.RolloutInterface { return rollouts(c, o.Namespace) } +func (o *rollout) GetGroupResource() schema.GroupResource { + return schema.GroupResource{ + Group: o.TypeMeta.GroupVersionKind().Group, + Resource: "rollouts", + } +} + func (o *rollout) GetKind() Kind { return RolloutKind } @@ -377,6 +392,13 @@ func (o *replicaSet) ki(c context.Context) typedApps.ReplicaSetInterface { return replicaSets(c, o.Namespace) } +func (o *replicaSet) GetGroupResource() schema.GroupResource { + return schema.GroupResource{ + Group: o.TypeMeta.GroupVersionKind().Group, + Resource: "replicasets", + } +} + func (o *replicaSet) GetKind() Kind { return ReplicaSetKind } @@ -442,6 +464,13 @@ func (o *statefulSet) ki(c context.Context) typedApps.StatefulSetInterface { return statefulSets(c, o.Namespace) } +func (o *statefulSet) GetGroupResource() schema.GroupResource { + return schema.GroupResource{ + Group: o.TypeMeta.GroupVersionKind().Group, + Resource: "statefulsets", + } +} + func (o *statefulSet) GetKind() Kind { return StatefulSetKind } From 9007abeaa3e05677794e99ca66ac3dd7d68d53d1 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 1 Feb 2025 17:42:54 +0100 Subject: [PATCH 39/61] Update module dependencies. Signed-off-by: Thomas Hallgren --- DEPENDENCIES.md | 14 +++++++------- go.mod | 14 +++++++------- go.sum | 27 ++++++++++++++------------- pkg/vif/device_linux.go | 9 ++++++++- pkg/vif/testdata/router/go.mod | 8 ++++---- pkg/vif/testdata/router/go.sum | 15 ++++++++------- pkg/vif/tunneling_device.go | 15 ++++++++++++++- rpc/go.mod | 2 +- rpc/go.sum | 4 ++-- tools/src/go-mkopensource/go.mod | 6 +++--- tools/src/go-mkopensource/go.sum | 6 ++++++ 11 files changed, 74 insertions(+), 46 deletions(-) diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index b1b9a21843..9083b551bf 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -25,7 +25,7 @@ following Free and Open Source software: github.com/containerd/log v0.1.0 Apache License 2.0 github.com/containerd/platforms v0.2.1 Apache License 2.0 github.com/coreos/go-iptables v0.8.0 Apache License 2.0 - github.com/cyphar/filepath-securejoin v0.4.0 3-clause BSD license + github.com/cyphar/filepath-securejoin v0.4.1 3-clause BSD license github.com/datawire/argo-rollouts-go-client v0.0.0-20241216133646-cb1073556c99 Apache License 2.0 github.com/datawire/dlib v1.3.1 Apache License 2.0 github.com/datawire/dtest v0.0.0-20210928162311-722b199c4c2f Apache License 2.0 @@ -41,7 +41,7 @@ following Free and Open Source software: github.com/docker/go-metrics v0.0.1 Apache License 2.0 github.com/docker/go-units v0.5.0 Apache License 2.0 github.com/emicklei/go-restful/v3 v3.12.1 MIT license - github.com/evanphx/json-patch v5.9.0+incompatible 3-clause BSD license + github.com/evanphx/json-patch v5.9.11+incompatible 3-clause BSD license github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f MIT license github.com/fatih/camelcase v1.0.0 MIT license github.com/fatih/color v1.18.0 MIT license @@ -52,7 +52,7 @@ following Free and Open Source software: github.com/fxamacker/cbor/v2 v2.7.0 MIT license github.com/go-errors/errors v1.5.1 MIT license github.com/go-gorp/gorp/v3 v3.1.0 MIT license - github.com/go-json-experiment/json v0.0.0-20250124004741-3d76ae074650 3-clause BSD license + github.com/go-json-experiment/json v0.0.0-20250129011340-4e0381018ad6 3-clause BSD license github.com/go-logr/logr v1.4.2 Apache License 2.0 github.com/go-logr/stdr v1.2.2 Apache License 2.0 github.com/go-openapi/jsonpointer v0.21.0 Apache License 2.0 @@ -126,9 +126,9 @@ following Free and Open Source software: github.com/spf13/afero v1.12.0 Apache License 2.0 github.com/spf13/cast v1.7.1 MIT license github.com/spf13/cobra v1.8.1 Apache License 2.0 - github.com/spf13/pflag v1.0.5 3-clause BSD license + github.com/spf13/pflag v1.0.6 3-clause BSD license github.com/stretchr/testify v1.10.0 MIT license - github.com/telepresenceio/go-fuseftp/rpc v0.5.0 Apache License 2.0 + github.com/telepresenceio/go-fuseftp/rpc v0.6.1 Apache License 2.0 github.com/telepresenceio/telepresence/rpc/v2 (modified) Apache License 2.0 github.com/vishvananda/netlink v1.3.0 Apache License 2.0 github.com/vishvananda/netns v0.0.5 Apache License 2.0 @@ -155,13 +155,13 @@ following Free and Open Source software: golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 MIT license golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 MIT license golang.zx2c4.com/wireguard/windows v0.5.3 MIT license - google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 Apache License 2.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 Apache License 2.0 google.golang.org/grpc v1.70.0 Apache License 2.0 google.golang.org/protobuf v1.36.4 3-clause BSD license gopkg.in/evanphx/json-patch.v4 v4.12.0 3-clause BSD license gopkg.in/inf.v0 v0.9.1 3-clause BSD license gopkg.in/yaml.v3 v3.0.1 Apache License 2.0, MIT license - gvisor.dev/gvisor v0.0.0-20250115195935-26653e7d8816 Apache License 2.0, MIT license + gvisor.dev/gvisor v0.0.0-20250131185017-b744a1bd640b Apache License 2.0, MIT license helm.sh/helm/v3 v3.17.0 Apache License 2.0 k8s.io/api v0.32.1 Apache License 2.0 k8s.io/apiextensions-apiserver v0.32.1 Apache License 2.0 diff --git a/go.mod b/go.mod index 75f95d7191..5862d80a97 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/docker/docker v27.5.1+incompatible github.com/docker/go-connections v0.5.0 github.com/fsnotify/fsnotify v1.8.0 - github.com/go-json-experiment/json v0.0.0-20250124004741-3d76ae074650 + github.com/go-json-experiment/json v0.0.0-20250129011340-4e0381018ad6 github.com/godbus/dbus/v5 v5.1.0 github.com/golang/mock v1.7.0-rc.1 github.com/google/go-cmp v0.6.0 @@ -33,9 +33,9 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.12.0 github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 + github.com/spf13/pflag v1.0.6 github.com/stretchr/testify v1.10.0 - github.com/telepresenceio/go-fuseftp/rpc v0.5.0 + github.com/telepresenceio/go-fuseftp/rpc v0.6.1 github.com/telepresenceio/telepresence/rpc/v2 v2.21.1 github.com/vishvananda/netlink v1.3.0 golang.org/x/net v0.34.0 @@ -46,7 +46,7 @@ require ( google.golang.org/grpc v1.70.0 google.golang.org/protobuf v1.36.4 gopkg.in/yaml.v3 v3.0.1 - gvisor.dev/gvisor v0.0.0-20250115195935-26653e7d8816 + gvisor.dev/gvisor v0.0.0-20250131185017-b744a1bd640b helm.sh/helm/v3 v3.17.0 k8s.io/api v0.32.1 k8s.io/apimachinery v0.32.1 @@ -76,7 +76,7 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect - github.com/cyphar/filepath-securejoin v0.4.0 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/cli v27.5.1+incompatible // indirect @@ -85,7 +85,7 @@ require ( github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect - github.com/evanphx/json-patch v5.9.0+incompatible // indirect + github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fatih/camelcase v1.0.0 // indirect github.com/fatih/color v1.18.0 // indirect @@ -171,7 +171,7 @@ require ( golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.29.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/apiextensions-apiserver v0.32.1 // indirect diff --git a/go.sum b/go.sum index a78554aeb9..4b587db3bf 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.4.0 h1:PioTG9TBRSApBpYGnDU8HC+miIsX8vitBH9LGNNMoLQ= -github.com/cyphar/filepath-securejoin v0.4.0/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= +github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/datawire/argo-rollouts-go-client v0.0.0-20241216133646-cb1073556c99 h1:a+yPIx3r59bp9OnMM/CMgCleWGreM9bfIHPUatvlMJk= github.com/datawire/argo-rollouts-go-client v0.0.0-20241216133646-cb1073556c99/go.mod h1:2O7ijdMXY8T19xQXtgMYQYJwWgudnB9n358O00YqSms= github.com/datawire/dlib v1.2.4-0.20210629021142-e221f3b9c3b8/go.mod h1:OdrErY06tawcmEkhTLeb1k3IN2HyzT3zcW4DsqQsJOM= @@ -107,8 +107,8 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arX github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= -github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= +github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= @@ -133,8 +133,8 @@ github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8b github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= -github.com/go-json-experiment/json v0.0.0-20250124004741-3d76ae074650 h1:J+1sdh5qqr2BcGbX0BHAr3W770pGck424xlY9OlIrh4= -github.com/go-json-experiment/json v0.0.0-20250124004741-3d76ae074650/go.mod h1:BWmvoE1Xia34f3l/ibJweyhrT+aROb/FQ6d+37F0e2s= +github.com/go-json-experiment/json v0.0.0-20250129011340-4e0381018ad6 h1:7zwULZ01CYYD6IIVPQPO+umwvqbymTboqiQOc21KUgo= +github.com/go-json-experiment/json v0.0.0-20250129011340-4e0381018ad6/go.mod h1:BWmvoE1Xia34f3l/ibJweyhrT+aROb/FQ6d+37F0e2s= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= @@ -381,8 +381,9 @@ github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -396,8 +397,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/telepresenceio/go-fuseftp/rpc v0.5.0 h1:OruBUIe2LcOuNX+BTATZL4v8JjlA0d3foM8i8wv4FrE= -github.com/telepresenceio/go-fuseftp/rpc v0.5.0/go.mod h1:vA6l+nLsyVHs3Wu1r9BQEO4+ZhL2sCZP+Jmqptdnn9E= +github.com/telepresenceio/go-fuseftp/rpc v0.6.1 h1:y/RA6OiE6qM47SiBeSaez+a+PVdIUEl68M7Wl+SqJcc= +github.com/telepresenceio/go-fuseftp/rpc v0.6.1/go.mod h1:jLvPHOWARcSRV5b1zxRYbv4Sa5VlMZxpnyTpCKrlPzA= github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk= github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= @@ -561,8 +562,8 @@ golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prr google.golang.org/genproto v0.0.0-20241209162323-e6fa225c2576 h1:k48HcZ4FE6in0o8IflZCkc1lTc2u37nhGd8P+fo4r24= google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 h1:91mG8dNTpkC0uChJUQ9zCiRqx3GEEFOWaRZ0mI6Oj2I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 h1:J1H9f+LEdWAfHcez/4cvaVBox7cOYT+IU6rgqj5x++8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= @@ -584,8 +585,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -gvisor.dev/gvisor v0.0.0-20250115195935-26653e7d8816 h1:ZIPfC6hWCapUHskzT+Hqq648v+os3ZnruhEA0NNQsQc= -gvisor.dev/gvisor v0.0.0-20250115195935-26653e7d8816/go.mod h1:5DMfjtclAbTIjbXqO1qCe2K5GKKxWz2JHvCChuTcJEM= +gvisor.dev/gvisor v0.0.0-20250131185017-b744a1bd640b h1:stbYFdh0Ua76MFQ4oNnCo3IAoW2+iP1xJC1QUzXBIzI= +gvisor.dev/gvisor v0.0.0-20250131185017-b744a1bd640b/go.mod h1:5DMfjtclAbTIjbXqO1qCe2K5GKKxWz2JHvCChuTcJEM= helm.sh/helm/v3 v3.17.0 h1:DUD4AGdNVn7PSTYfxe1gmQG7s18QeWv/4jI9TubnhT0= helm.sh/helm/v3 v3.17.0/go.mod h1:Mo7eGyKPPHlS0Ml67W8z/lbkox/gD9Xt1XpD6bxvZZA= k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= diff --git a/pkg/vif/device_linux.go b/pkg/vif/device_linux.go index ddf6ad4262..6c7928808e 100644 --- a/pkg/vif/device_linux.go +++ b/pkg/vif/device_linux.go @@ -19,6 +19,7 @@ const devicePath = "/dev/net/tun" type device struct { fd int name string + endPoint stack.LinkEndpoint interfaceIndex uint32 } @@ -117,14 +118,20 @@ func (d *device) createLinkEndpoint() (stack.LinkEndpoint, error) { if err != nil { return nil, err } - return fdbased.New(&fdbased.Options{ + ep, err := fdbased.New(&fdbased.Options{ FDs: []int{d.fd}, MTU: mtu, PacketDispatchMode: fdbased.RecvMMsg, }) + if err != nil { + return nil, err + } + d.endPoint = ep + return ep, nil } func (d *device) Close() { + d.endPoint.Close() _ = unix.Close(d.fd) } diff --git a/pkg/vif/testdata/router/go.mod b/pkg/vif/testdata/router/go.mod index 2225f3ef47..57ab5277fb 100644 --- a/pkg/vif/testdata/router/go.mod +++ b/pkg/vif/testdata/router/go.mod @@ -18,7 +18,7 @@ require ( github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-errors/errors v1.5.1 // indirect - github.com/go-json-experiment/json v0.0.0-20250124004741-3d76ae074650 // indirect + github.com/go-json-experiment/json v0.0.0-20250129011340-4e0381018ad6 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -48,7 +48,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/telepresenceio/telepresence/rpc/v2 v2.21.1 // indirect github.com/vishvananda/netlink v1.3.0 // indirect github.com/vishvananda/netns v0.0.5 // indirect @@ -64,13 +64,13 @@ require ( golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect golang.zx2c4.com/wireguard/windows v0.5.3 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 // indirect google.golang.org/grpc v1.70.0 // indirect google.golang.org/protobuf v1.36.4 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gvisor.dev/gvisor v0.0.0-20250115195935-26653e7d8816 // indirect + gvisor.dev/gvisor v0.0.0-20250131185017-b744a1bd640b // indirect k8s.io/api v0.32.1 // indirect k8s.io/apimachinery v0.32.1 // indirect k8s.io/cli-runtime v0.32.1 // indirect diff --git a/pkg/vif/testdata/router/go.sum b/pkg/vif/testdata/router/go.sum index e428488955..db1233a145 100644 --- a/pkg/vif/testdata/router/go.sum +++ b/pkg/vif/testdata/router/go.sum @@ -23,8 +23,8 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-json-experiment/json v0.0.0-20250124004741-3d76ae074650 h1:J+1sdh5qqr2BcGbX0BHAr3W770pGck424xlY9OlIrh4= -github.com/go-json-experiment/json v0.0.0-20250124004741-3d76ae074650/go.mod h1:BWmvoE1Xia34f3l/ibJweyhrT+aROb/FQ6d+37F0e2s= +github.com/go-json-experiment/json v0.0.0-20250129011340-4e0381018ad6 h1:7zwULZ01CYYD6IIVPQPO+umwvqbymTboqiQOc21KUgo= +github.com/go-json-experiment/json v0.0.0-20250129011340-4e0381018ad6/go.mod h1:BWmvoE1Xia34f3l/ibJweyhrT+aROb/FQ6d+37F0e2s= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -113,8 +113,9 @@ github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= @@ -198,8 +199,8 @@ golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uI golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA= golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE= golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 h1:91mG8dNTpkC0uChJUQ9zCiRqx3GEEFOWaRZ0mI6Oj2I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 h1:J1H9f+LEdWAfHcez/4cvaVBox7cOYT+IU6rgqj5x++8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= @@ -214,8 +215,8 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gvisor.dev/gvisor v0.0.0-20250115195935-26653e7d8816 h1:ZIPfC6hWCapUHskzT+Hqq648v+os3ZnruhEA0NNQsQc= -gvisor.dev/gvisor v0.0.0-20250115195935-26653e7d8816/go.mod h1:5DMfjtclAbTIjbXqO1qCe2K5GKKxWz2JHvCChuTcJEM= +gvisor.dev/gvisor v0.0.0-20250131185017-b744a1bd640b h1:stbYFdh0Ua76MFQ4oNnCo3IAoW2+iP1xJC1QUzXBIzI= +gvisor.dev/gvisor v0.0.0-20250131185017-b744a1bd640b/go.mod h1:5DMfjtclAbTIjbXqO1qCe2K5GKKxWz2JHvCChuTcJEM= k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= diff --git a/pkg/vif/tunneling_device.go b/pkg/vif/tunneling_device.go index 31c1d359e4..206fc6ef43 100644 --- a/pkg/vif/tunneling_device.go +++ b/pkg/vif/tunneling_device.go @@ -2,6 +2,7 @@ package vif import ( "context" + "time" "github.com/hashicorp/go-multierror" "gvisor.dev/gvisor/pkg/tcpip/stack" @@ -65,6 +66,18 @@ func (vif *TunnelingDevice) Run(ctx context.Context) (err error) { dlog.Debug(ctx, "vif ended") }() - vif.stack.Wait() + // The stack.Wait gets stuck at times, even though the stack is closed, so we ensure that the + // Run terminates anyway here. + // See https://github.com/google/gvisor/issues/11456 + stackDoneCh := make(chan struct{}) + go func() { + vif.stack.Wait() + close(stackDoneCh) + }() + <-ctx.Done() + select { + case <-stackDoneCh: + case <-time.After(2 * time.Second): + } return nil } diff --git a/rpc/go.mod b/rpc/go.mod index 6b5437a77d..7668e09a9f 100644 --- a/rpc/go.mod +++ b/rpc/go.mod @@ -11,5 +11,5 @@ require ( golang.org/x/net v0.34.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 // indirect ) diff --git a/rpc/go.sum b/rpc/go.sum index 3ea53dd9e1..803f8b554c 100644 --- a/rpc/go.sum +++ b/rpc/go.sum @@ -24,8 +24,8 @@ golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 h1:91mG8dNTpkC0uChJUQ9zCiRqx3GEEFOWaRZ0mI6Oj2I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 h1:J1H9f+LEdWAfHcez/4cvaVBox7cOYT+IU6rgqj5x++8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= diff --git a/tools/src/go-mkopensource/go.mod b/tools/src/go-mkopensource/go.mod index f9c4f9a928..0ef10704d6 100644 --- a/tools/src/go-mkopensource/go.mod +++ b/tools/src/go-mkopensource/go.mod @@ -11,7 +11,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.1.5 // indirect github.com/cloudflare/circl v1.5.0 // indirect - github.com/cyphar/filepath-securejoin v0.4.0 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.2 // indirect @@ -22,8 +22,8 @@ require ( github.com/mmcloughlin/avo v0.6.0 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect - github.com/skeema/knownhosts v1.3.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect golang.org/x/crypto v0.32.0 // indirect golang.org/x/mod v0.22.0 // indirect diff --git a/tools/src/go-mkopensource/go.sum b/tools/src/go-mkopensource/go.sum index 5bd44f8899..1a88c98c25 100644 --- a/tools/src/go-mkopensource/go.sum +++ b/tools/src/go-mkopensource/go.sum @@ -21,6 +21,8 @@ github.com/cyphar/filepath-securejoin v0.3.6 h1:4d9N5ykBnSp5Xn2JkhocYDkOpURL/18C github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/cyphar/filepath-securejoin v0.4.0 h1:PioTG9TBRSApBpYGnDU8HC+miIsX8vitBH9LGNNMoLQ= github.com/cyphar/filepath-securejoin v0.4.0/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= +github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/datawire/go-mkopensource v0.0.13 h1:Di8P2jXdGR34HFDb/axqgLxMv0t1UHeR9KtzXi+rutI= github.com/datawire/go-mkopensource v0.0.13/go.mod h1:Gz65lrm4uYY1aBhxWqjwg6DUKFdBCHfpbGC9D2DDuhQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -87,8 +89,12 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY= github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= From b82a5b77174e243d03cc0c7e3f07647f1910a294 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Mon, 3 Feb 2025 10:41:43 +0100 Subject: [PATCH 40/61] Add build flag to link the go-fuseftp implementation. This build-flag is not enabled by default, but can be used when there is a desire to build a telepresence client that links the fuseftp implementation instead of using a remote process via gRPC. Signed-off-by: Thomas Hallgren --- DEPENDENCIES.md | 3 + build-aux/main.mk | 40 +++++---- go.mod | 3 + go.sum | 6 ++ pkg/client/remotefs/fuseftp.go | 2 +- pkg/client/remotefs/fuseftp_docker.go | 15 ++-- .../{fuseftpembed.go => fuseftp_embedded.go} | 2 +- .../{fuseftpextern.go => fuseftp_external.go} | 2 +- .../remotefs/{ftp.go => fuseftp_grpc.go} | 2 + pkg/client/remotefs/fuseftp_linked.go | 87 +++++++++++++++++++ pkg/client/userd/trafficmgr/mount.go | 3 +- pkg/grpc/server/server.go | 2 +- 12 files changed, 140 insertions(+), 27 deletions(-) rename pkg/client/remotefs/{fuseftpembed.go => fuseftp_embedded.go} (90%) rename pkg/client/remotefs/{fuseftpextern.go => fuseftp_external.go} (83%) rename pkg/client/remotefs/{ftp.go => fuseftp_grpc.go} (98%) create mode 100644 pkg/client/remotefs/fuseftp_linked.go diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 9083b551bf..186f2f02e8 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -79,6 +79,7 @@ following Free and Open Source software: github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb MIT license github.com/huandu/xstrings v1.5.0 MIT license github.com/inconshreveable/mousetrap v1.1.0 Apache License 2.0 + github.com/jlaffaye/ftp v0.2.0 ISC license github.com/jmoiron/sqlx v1.4.0 MIT license github.com/josharian/intern v1.0.1-0.20211109044230-42b52b674af5 MIT license github.com/json-iterator/go v1.1.12 MIT license @@ -128,10 +129,12 @@ following Free and Open Source software: github.com/spf13/cobra v1.8.1 Apache License 2.0 github.com/spf13/pflag v1.0.6 3-clause BSD license github.com/stretchr/testify v1.10.0 MIT license + github.com/telepresenceio/go-fuseftp v0.6.1 Apache License 2.0 github.com/telepresenceio/go-fuseftp/rpc v0.6.1 Apache License 2.0 github.com/telepresenceio/telepresence/rpc/v2 (modified) Apache License 2.0 github.com/vishvananda/netlink v1.3.0 Apache License 2.0 github.com/vishvananda/netns v0.0.5 Apache License 2.0 + github.com/winfsp/cgofuse v1.6.0 MIT license github.com/x448/float16 v0.8.4 MIT license github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb Apache License 2.0 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 Apache License 2.0 diff --git a/build-aux/main.mk b/build-aux/main.mk index eb028fd2a1..eba0a33200 100644 --- a/build-aux/main.mk +++ b/build-aux/main.mk @@ -34,6 +34,9 @@ export DOCKER_BUILDKIT := 1 .PHONY: FORCE FORCE: +EXTERNAL_FUSEFTP=0 +LINKED_FUSEFTP=0 + # Build with CGO_ENABLED=0 on all platforms to ensure that the binary is as # portable as possible, but we must make an exception for darwin, because # the Go implementation of the DNS resolver doesn't work properly there unless @@ -41,8 +44,13 @@ FORCE: ifeq ($(GOOS),darwin) CGO_ENABLED=1 else +ifeq ($(GOOS),linux) +# The winfsp module requires CGO on Linux. +CGO_ENABLED=$(LINKED_FUSEFTP) +else CGO_ENABLED=0 endif +endif ifeq ($(GOOS),windows) BEXE=.exe @@ -52,8 +60,6 @@ BEXE= BZIP= endif -EMBED_FUSEFTP=1 - # Generate: artifacts that get checked in to Git # ============================================== @@ -164,10 +170,19 @@ else sdkroot= endif +BUILD_TAGS= +build-deps: + ifeq ($(DOCKER_BUILD),1) +BUILD_TAGS=-tags docker +else +ifeq ($(EXTERNAL_FUSEFTP),1) +BUILD_TAGS=-tags external_fuseftp build-deps: else -ifeq ($(EMBED_FUSEFTP),1) +ifeq ($(LINKED_FUSEFTP),1) +BUILD_TAGS=-tags linked_fuseftp +else FUSEFTP_VERSION=$(shell go list -m -f {{.Version}} github.com/telepresenceio/go-fuseftp/rpc) $(BUILDDIR)/fuseftp-$(GOOS)-$(GOARCH)$(BEXE): go.mod @@ -178,8 +193,7 @@ pkg/client/remotefs/fuseftp.bits: $(BUILDDIR)/fuseftp-$(GOOS)-$(GOARCH)$(BEXE) F cp $< $@ build-deps: pkg/client/remotefs/fuseftp.bits -else -build-deps: +endif endif endif @@ -211,14 +225,10 @@ $(TELEPRESENCE): build-deps $(BINDIR)/wintun.dll FORCE endif mkdir -p $(@D) ifeq ($(DOCKER_BUILD),1) - CGO_ENABLED=$(CGO_ENABLED) $(sdkroot) go build -tags docker -trimpath -ldflags=-X=$(PKG_VERSION).Version=$(TELEPRESENCE_VERSION) -o $@ ./cmd/telepresence + CGO_ENABLED=$(CGO_ENABLED) $(sdkroot) go build $(BUILD_TAGS) -trimpath -ldflags=-X=$(PKG_VERSION).Version=$(TELEPRESENCE_VERSION) -o $@ ./cmd/telepresence else # -buildmode=pie addresses https://github.com/datawire/telepresence2-proprietary/issues/315 -ifeq ($(EMBED_FUSEFTP),1) - CGO_ENABLED=$(CGO_ENABLED) $(sdkroot) go build -tags embed_fuseftp -buildmode=pie -trimpath -ldflags=-X=$(PKG_VERSION).Version=$(TELEPRESENCE_VERSION) -o $@ ./cmd/telepresence -else - CGO_ENABLED=$(CGO_ENABLED) $(sdkroot) go build -buildmode=pie -trimpath -ldflags=-X=$(PKG_VERSION).Version=$(TELEPRESENCE_VERSION) -o $@ ./cmd/telepresence -endif + CGO_ENABLED=$(CGO_ENABLED) $(sdkroot) go build $(BUILD_TAGS) -buildmode=pie -trimpath -ldflags=-X=$(PKG_VERSION).Version=$(TELEPRESENCE_VERSION) -o $@ ./cmd/telepresence endif ifeq ($(GOOS),windows) @@ -378,7 +388,7 @@ shellscripts += ./packaging/windows-package.sh lint: lint-rpc lint-go -GOLANGCI_VERSION:=v1.62.2 +GOLANGCI_VERSION:=v1.63.4 lint-go: lint-deps ## (QA) Run the golangci-lint $(eval badimports = $(shell find cmd integration_test pkg -name '*.go' | grep -v '/mocks/' | xargs $(tools/gosimports) --local github.com/datawire/,github.com/telepresenceio/ -l)) @@ -434,11 +444,7 @@ endif # is only used for extensions. Therefore, we want to validate that our tests, and # telepresence, run without requiring any outside dependencies. set -o pipefail -ifeq ($(EMBED_FUSEFTP),1) - TELEPRESENCE_MAX_LOGFILES=300 TELEPRESENCE_LOGIN_DOMAIN=127.0.0.1 CGO_ENABLED=$(CGO_ENABLED) go test -tags embed_fuseftp -failfast -json -timeout=80m ./integration_test/... | $(tools/test-report) -else - TELEPRESENCE_MAX_LOGFILES=300 TELEPRESENCE_LOGIN_DOMAIN=127.0.0.1 CGO_ENABLED=$(CGO_ENABLED) go test -failfast -json -timeout=80m ./integration_test/... | $(tools/test-report) -endif + TELEPRESENCE_MAX_LOGFILES=300 TELEPRESENCE_LOGIN_DOMAIN=127.0.0.1 CGO_ENABLED=$(CGO_ENABLED) go test $(BUILD_TAGS) -failfast -json -timeout=80m ./integration_test/... | $(tools/test-report) .PHONY: _login _login: diff --git a/go.mod b/go.mod index 5862d80a97..4a93d61591 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.6 github.com/stretchr/testify v1.10.0 + github.com/telepresenceio/go-fuseftp v0.6.1 github.com/telepresenceio/go-fuseftp/rpc v0.6.1 github.com/telepresenceio/telepresence/rpc/v2 v2.21.1 github.com/vishvananda/netlink v1.3.0 @@ -114,6 +115,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jlaffaye/ftp v0.2.0 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect github.com/josharian/intern v1.0.1-0.20211109044230-42b52b674af5 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -153,6 +155,7 @@ require ( github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/vishvananda/netns v0.0.5 // indirect + github.com/winfsp/cgofuse v1.6.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect diff --git a/go.sum b/go.sum index 4b587db3bf..ed04dc4a9b 100644 --- a/go.sum +++ b/go.sum @@ -222,6 +222,8 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= +github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josharian/intern v1.0.1-0.20211109044230-42b52b674af5 h1:f8m7k2T128wwQej7ewBVgUfHNgCu3uXod6wopWGDvE4= @@ -397,6 +399,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/telepresenceio/go-fuseftp v0.6.1 h1:v3Hqwl2rDxep3kHfEOwsXijOOCXT3blRGN0cs45uR9A= +github.com/telepresenceio/go-fuseftp v0.6.1/go.mod h1:EXH+oL5GalxZ/pZV5MCHFYW3aLW1SRxV5wSedWoSS2g= github.com/telepresenceio/go-fuseftp/rpc v0.6.1 h1:y/RA6OiE6qM47SiBeSaez+a+PVdIUEl68M7Wl+SqJcc= github.com/telepresenceio/go-fuseftp/rpc v0.6.1/go.mod h1:jLvPHOWARcSRV5b1zxRYbv4Sa5VlMZxpnyTpCKrlPzA= github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk= @@ -404,6 +408,8 @@ github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/winfsp/cgofuse v1.6.0 h1:re3W+HTd0hj4fISPBqfsrwyvPFpzqhDu8doJ9nOPDB0= +github.com/winfsp/cgofuse v1.6.0/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/pkg/client/remotefs/fuseftp.go b/pkg/client/remotefs/fuseftp.go index a77ee48577..5e95a60737 100644 --- a/pkg/client/remotefs/fuseftp.go +++ b/pkg/client/remotefs/fuseftp.go @@ -1,4 +1,4 @@ -//go:build !docker +//go:build !docker && !linked_fuseftp package remotefs diff --git a/pkg/client/remotefs/fuseftp_docker.go b/pkg/client/remotefs/fuseftp_docker.go index f1833a8529..2b96c0e08d 100644 --- a/pkg/client/remotefs/fuseftp_docker.go +++ b/pkg/client/remotefs/fuseftp_docker.go @@ -1,30 +1,35 @@ //go:build docker -// +build docker package remotefs import ( "context" "errors" + "sync" "github.com/telepresenceio/go-fuseftp/rpc" ) +// NewFTPMounter returns nil. It's here to satisfy the linker. +func NewFTPMounter(rpc.FuseFTPClient, *sync.WaitGroup) Mounter { + return nil +} + type fuseFtpMgr struct{} type FuseFTPManager interface { - DeferInit(ctx context.Context) error - GetFuseFTPClient(ctx context.Context) rpc.FuseFTPClient + DeferInit(context.Context) error + GetFuseFTPClient(context.Context) rpc.FuseFTPClient } func NewFuseFTPManager() FuseFTPManager { return &fuseFtpMgr{} } -func (s *fuseFtpMgr) DeferInit(ctx context.Context) error { +func (s *fuseFtpMgr) DeferInit(context.Context) error { return errors.New("fuseftp client is not available") } -func (s *fuseFtpMgr) GetFuseFTPClient(ctx context.Context) rpc.FuseFTPClient { +func (s *fuseFtpMgr) GetFuseFTPClient(context.Context) rpc.FuseFTPClient { return nil } diff --git a/pkg/client/remotefs/fuseftpembed.go b/pkg/client/remotefs/fuseftp_embedded.go similarity index 90% rename from pkg/client/remotefs/fuseftpembed.go rename to pkg/client/remotefs/fuseftp_embedded.go index 3045aea491..caf23fcf5d 100644 --- a/pkg/client/remotefs/fuseftpembed.go +++ b/pkg/client/remotefs/fuseftp_embedded.go @@ -1,4 +1,4 @@ -//go:build embed_fuseftp && !docker +//go:build !(docker || external_fuseftp || linked_fuseftp) package remotefs diff --git a/pkg/client/remotefs/fuseftpextern.go b/pkg/client/remotefs/fuseftp_external.go similarity index 83% rename from pkg/client/remotefs/fuseftpextern.go rename to pkg/client/remotefs/fuseftp_external.go index 997cffd411..24cba4f6b4 100644 --- a/pkg/client/remotefs/fuseftpextern.go +++ b/pkg/client/remotefs/fuseftp_external.go @@ -1,4 +1,4 @@ -//go:build !embed_fuseftp && !docker +//go:build external_fuseftp && !docker package remotefs diff --git a/pkg/client/remotefs/ftp.go b/pkg/client/remotefs/fuseftp_grpc.go similarity index 98% rename from pkg/client/remotefs/ftp.go rename to pkg/client/remotefs/fuseftp_grpc.go index afc42380d1..43072da6de 100644 --- a/pkg/client/remotefs/ftp.go +++ b/pkg/client/remotefs/fuseftp_grpc.go @@ -1,3 +1,5 @@ +//go:build !(linked_fuseftp || docker) + package remotefs import ( diff --git a/pkg/client/remotefs/fuseftp_linked.go b/pkg/client/remotefs/fuseftp_linked.go new file mode 100644 index 0000000000..f446d597f1 --- /dev/null +++ b/pkg/client/remotefs/fuseftp_linked.go @@ -0,0 +1,87 @@ +//go:build linked_fuseftp && !docker + +package remotefs + +import ( + "context" + "net/netip" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" + + "github.com/datawire/dlib/dlog" + "github.com/telepresenceio/go-fuseftp/pkg/fs" + "github.com/telepresenceio/go-fuseftp/rpc" + "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" + "github.com/telepresenceio/telepresence/v2/pkg/client" +) + +type ftpMounter struct { + mountPoint string + cancel context.CancelFunc + ftpClient fs.FTPClient + iceptWG *sync.WaitGroup +} + +type fuseFtpMgr struct{} + +type FuseFTPManager interface { + DeferInit(ctx context.Context) error + GetFuseFTPClient(ctx context.Context) rpc.FuseFTPClient +} + +func NewFuseFTPManager() FuseFTPManager { + return &fuseFtpMgr{} +} + +func (s *fuseFtpMgr) DeferInit(_ context.Context) error { + return nil +} + +func (s *fuseFtpMgr) GetFuseFTPClient(_ context.Context) rpc.FuseFTPClient { + return rpc.NewFuseFTPClient(nil) +} + +func NewFTPMounter(_ rpc.FuseFTPClient, iceptWG *sync.WaitGroup) Mounter { + // The FTPClient uses the global logrus logger. It's very verbose on DebugLevel. + logrus.SetLevel(logrus.InfoLevel) + return &ftpMounter{iceptWG: iceptWG} +} + +func (m *ftpMounter) Start(ctx context.Context, workload, container, clientMountPoint, mountPoint string, podAddrPort netip.AddrPort, ro bool) error { + roTxt := "" + if ro { + roTxt = " read-only" + } + if m.ftpClient == nil { + cfg := client.GetConfig(ctx) + dlog.Infof(ctx, "Mounting FTP file system for container %s[%s] (address %s)%s at %q", workload, container, podAddrPort, roTxt, clientMountPoint) + // FTPs remote mount is already relative to the agentconfig.ExportsMountPoint + rmp := strings.TrimPrefix(mountPoint, agentconfig.ExportsMountPoint) + ftpClient, err := fs.NewFTPClient(ctx, podAddrPort, rmp, ro, cfg.Timeouts().Get(client.TimeoutFtpReadWrite)) + if err != nil { + return err + } + host := fs.NewHost(ftpClient, clientMountPoint) + if err = host.Start(ctx, 5*time.Second); err != nil { + return err + } + + m.ftpClient = ftpClient + // Ensure unmount when intercept context is cancelled + m.iceptWG.Add(1) + go func() { + defer m.iceptWG.Done() + <-ctx.Done() + dlog.Debugf(ctx, "Unmounting FTP file system for container %s[%s] (address %s) at %q", workload, container, podAddrPort, clientMountPoint) + }() + dlog.Infof(ctx, "File system for container %s[%s] (address %s) successfully mounted%s at %q", workload, container, podAddrPort, roTxt, clientMountPoint) + return nil + } + + // Assign a new address to the FTP client. This kills any open connections but leaves the FUSE driver intact + dlog.Infof(ctx, "Switching remote address to %s for FTP file system for workload container %s[%s] at %q", podAddrPort, workload, container, clientMountPoint) + return m.ftpClient.SetAddress(podAddrPort) +} diff --git a/pkg/client/userd/trafficmgr/mount.go b/pkg/client/userd/trafficmgr/mount.go index daeb6c89e4..71e5792428 100644 --- a/pkg/client/userd/trafficmgr/mount.go +++ b/pkg/client/userd/trafficmgr/mount.go @@ -44,7 +44,8 @@ func (pa *podAccess) startMount(ctx context.Context, iceptWG, podWG *sync.WaitGr } // The FTP mounter survives multiple starts for the same intercept. It just resets the address mountCtx = pa.ctx - if fuseftp = userd.GetService(ctx).FuseFTPMgr().GetFuseFTPClient(ctx); fuseftp == nil { + fuseftp = userd.GetService(ctx).FuseFTPMgr().GetFuseFTPClient(ctx) + if fuseftp == nil { dlog.Errorf(ctx, "Client is configured to perform remote mounts using FTP, but the fuseftp server was unable to start") return } diff --git a/pkg/grpc/server/server.go b/pkg/grpc/server/server.go index 83e8a9c00a..3964100a83 100644 --- a/pkg/grpc/server/server.go +++ b/pkg/grpc/server/server.go @@ -115,9 +115,9 @@ func Stop(ctx context.Context, svc *grpc.Server, maxTime time.Duration) { // if the context has soft-cancel enabled. The server's Stop function will be called if no soft-cancel is enabled or // when the GracefulStop doesn't finish until the Done channel of the hard context closed. func Wait(ctx context.Context, svc *grpc.Server) { + <-ctx.Done() hardCtx := dcontext.HardContext(ctx) if hardCtx != ctx { - <-ctx.Done() dead := make(chan struct{}) go func() { dlog.Debug(ctx, "Initiating soft shutdown") From 6ba1193864c229414c40f6d7b4594b5c9f7faac0 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Thu, 6 Feb 2025 07:18:33 +0100 Subject: [PATCH 41/61] Get rid of the namespace lock mechanism. No longer needed. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/manager/mutator/watcher.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index a4c4a54f3b..fd1ea91e90 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -171,7 +171,6 @@ type inactivation struct { type configWatcher struct { cancel context.CancelFunc agentConfigs *xsync.MapOf[string, map[string]agentconfig.SidecarExt] - nsLocks *xsync.MapOf[string, *sync.RWMutex] informers *xsync.MapOf[string, *informersWithCancel] inactivePods *xsync.MapOf[types.UID, inactivation] startedAt time.Time @@ -235,7 +234,6 @@ func (c *configWatcher) Store(sce agentconfig.SidecarExt) { func NewWatcher() Map { w := &configWatcher{ - nsLocks: xsync.NewMapOf[string, *sync.RWMutex](), informers: xsync.NewMapOf[string, *informersWithCancel](), inactivePods: xsync.NewMapOf[types.UID, inactivation](), agentConfigs: xsync.NewMapOf[string, map[string]agentconfig.SidecarExt](), @@ -337,13 +335,6 @@ func (c *configWatcher) OnDelete(context.Context, string, string) error { return nil } -func (c *configWatcher) getNamespaceLock(ns string) *sync.RWMutex { - lock, _ := c.nsLocks.LoadOrCompute(ns, func() *sync.RWMutex { - return &sync.RWMutex{} - }) - return lock -} - // Get returns the Sidecar configuration that for the given key and namespace. // If no configuration is found, this function returns nil, nil. // An error is only returned when the configmap holding the configuration could not be loaded for @@ -497,12 +488,7 @@ func (c *configWatcher) DeleteMapsAndRolloutAll(ctx context.Context) { } func (c *configWatcher) deleteMapsAndRolloutNS(ctx context.Context, ns string, iwc *informersWithCancel) { - lock := c.getNamespaceLock(ns) - lock.Lock() - defer func() { - c.nsLocks.Delete(ns) - c.informers.Delete(ns) - }() + defer c.informers.Delete(ns) dlog.Debugf(ctx, "Cancelling watchers for namespace %s", ns) for i := 0; i < watcherMax; i++ { @@ -511,7 +497,6 @@ func (c *configWatcher) deleteMapsAndRolloutNS(ctx context.Context, ns string, i } } iwc.cancel() - lock.Unlock() err := c.DeleteAllPodsWithConfig(ctx, ns) if err != nil { From 53309625ad97663ecbcf11969baac0c6909b2bbd Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Thu, 6 Feb 2025 07:19:31 +0100 Subject: [PATCH 42/61] Rename "delete" pod methods to "evict" to clarify. Signed-off-by: Thomas Hallgren --- .../cmd/manager/mutator/service_watcher.go | 2 +- cmd/traffic/cmd/manager/mutator/watcher.go | 41 +++++++++---------- .../cmd/manager/mutator/workload_watcher.go | 4 +- cmd/traffic/cmd/manager/state/intercept.go | 25 +++++------ cmd/traffic/cmd/manager/state/state.go | 4 +- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/cmd/traffic/cmd/manager/mutator/service_watcher.go b/cmd/traffic/cmd/manager/mutator/service_watcher.go index 309ddcb52b..d866f5ceb0 100644 --- a/cmd/traffic/cmd/manager/mutator/service_watcher.go +++ b/cmd/traffic/cmd/manager/mutator/service_watcher.go @@ -150,7 +150,7 @@ func (c *configWatcher) updateSvc(ctx context.Context, svc *core.Service, trustU ac = acn.AgentConfig() c.Store(acn) dlog.Debugf(ctx, "deleting pods with config mismatch for %s %s.%s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace) - err = c.DeletePodsWithConfigMismatch(ctx, acn) + err = c.EvictPodsWithAgentConfigMismatch(ctx, acn) if err != nil { dlog.Error(ctx, err) } diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index fd1ea91e90..b0ad6c0a33 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "slices" - "sync" "sync/atomic" "time" @@ -41,9 +40,9 @@ type Map interface { DeleteMapsAndRolloutAll(context.Context) IsInactive(podID types.UID) bool Inactivate(podID types.UID) - DeletePodsWithConfig(ctx context.Context, wl k8sapi.Workload) error - DeletePodsWithConfigMismatch(ctx context.Context, scx agentconfig.SidecarExt) error - DeleteAllPodsWithConfig(ctx context.Context, namespace string) error + EvictPodsWithAgentConfig(ctx context.Context, wl k8sapi.Workload) error + EvictPodsWithAgentConfigMismatch(ctx context.Context, scx agentconfig.SidecarExt) error + EvictAllPodsWithAgentConfig(ctx context.Context, namespace string) error RegenerateAgentMaps(ctx context.Context, s string) error @@ -135,7 +134,7 @@ func (c *configWatcher) regenerateAgentMaps(ctx context.Context, ns string, gc a } if newSce == nil || !cmp.Equal(newSce, sce, dbpCmp) { go func() { - deletePod(ctx, pod) + evictPod(ctx, pod) }() } } @@ -498,13 +497,13 @@ func (c *configWatcher) deleteMapsAndRolloutNS(ctx context.Context, ns string, i } iwc.cancel() - err := c.DeleteAllPodsWithConfig(ctx, ns) + err := c.EvictAllPodsWithAgentConfig(ctx, ns) if err != nil { dlog.Errorf(ctx, "unable to delete agents in namespace %s: %v", ns, err) } } -func (c *configWatcher) DeletePodsWithConfigMismatch(ctx context.Context, scx agentconfig.SidecarExt) error { +func (c *configWatcher) EvictPodsWithAgentConfigMismatch(ctx context.Context, scx agentconfig.SidecarExt) error { ac := scx.AgentConfig() pods, err := podList(ctx, ac.WorkloadKind, ac.AgentName, ac.Namespace) if err != nil { @@ -516,7 +515,7 @@ func (c *configWatcher) DeletePodsWithConfigMismatch(ctx context.Context, scx ag } for _, pod := range pods { - err = c.DeleteIfMismatch(ctx, pod, cfgJSON) + err = c.evictPodWithAgentConfigMismatch(ctx, pod, cfgJSON) if err != nil { return err } @@ -524,14 +523,14 @@ func (c *configWatcher) DeletePodsWithConfigMismatch(ctx context.Context, scx ag return nil } -func (c *configWatcher) DeletePodsWithConfig(ctx context.Context, wl k8sapi.Workload) error { +func (c *configWatcher) EvictPodsWithAgentConfig(ctx context.Context, wl k8sapi.Workload) error { pods, err := podList(ctx, wl.GetKind(), wl.GetName(), wl.GetNamespace()) if err != nil { return err } for _, pod := range pods { - err = c.DeleteIfMismatch(ctx, pod, "") + err = c.evictPodWithAgentConfigMismatch(ctx, pod, "") if err != nil { return err } @@ -539,14 +538,14 @@ func (c *configWatcher) DeletePodsWithConfig(ctx context.Context, wl k8sapi.Work return nil } -func (c *configWatcher) DeleteAllPodsWithConfig(ctx context.Context, namespace string) error { +func (c *configWatcher) EvictAllPodsWithAgentConfig(ctx context.Context, namespace string) error { c.agentConfigs.Delete(namespace) pods, err := podList(ctx, "", "", namespace) if err != nil { return err } for _, pod := range pods { - err = c.DeleteIfMismatch(ctx, pod, "") + err = c.evictPodWithAgentConfigMismatch(ctx, pod, "") if err != nil { return err } @@ -554,9 +553,9 @@ func (c *configWatcher) DeleteAllPodsWithConfig(ctx context.Context, namespace s return nil } -func (c *configWatcher) DeleteIfMismatch(ctx context.Context, pod *core.Pod, cfgJSON string) error { +func (c *configWatcher) evictPodWithAgentConfigMismatch(ctx context.Context, pod *core.Pod, cfgJSON string) error { podID := pod.UID - if c.IsDeleted(podID) { + if c.isEvicted(podID) { dlog.Debugf(ctx, "Skipping pod %s because it is already deleted", pod.Name) return nil } @@ -575,13 +574,13 @@ func (c *configWatcher) DeleteIfMismatch(ctx context.Context, pod *core.Pod, cfg dlog.Debugf(ctx, "Skipping pod %s because it was deleted by another goroutine", pod.Name) return v, false } - deletePod(ctx, pod) + evictPod(ctx, pod) return inactivation{Time: time.Now(), deleted: true}, false }) return err } -func deletePod(ctx context.Context, pod *core.Pod) { +func evictPod(ctx context.Context, pod *core.Pod) { dlog.Debugf(ctx, "Evicting pod %s", pod.Name) err := k8sapi.GetK8sInterface(ctx).CoreV1().Pods(pod.Namespace).EvictV1(ctx, &v1.Eviction{ ObjectMeta: meta.ObjectMeta{Name: pod.Name, Namespace: pod.Namespace}, @@ -597,16 +596,16 @@ func (c *configWatcher) Inactivate(podID types.UID) { }) } -func (c *configWatcher) IsDeleted(podID types.UID) bool { - v, ok := c.inactivePods.Load(podID) - return ok && v.deleted -} - func (c *configWatcher) IsInactive(podID types.UID) bool { _, ok := c.inactivePods.Load(podID) return ok } +func (c *configWatcher) isEvicted(podID types.UID) bool { + v, ok := c.inactivePods.Load(podID) + return ok && v.deleted +} + func podIsRunning(pod *core.Pod) bool { switch pod.Status.Phase { case core.PodPending, core.PodRunning: diff --git a/cmd/traffic/cmd/manager/mutator/workload_watcher.go b/cmd/traffic/cmd/manager/mutator/workload_watcher.go index 4785b042e5..0f565a023c 100644 --- a/cmd/traffic/cmd/manager/mutator/workload_watcher.go +++ b/cmd/traffic/cmd/manager/mutator/workload_watcher.go @@ -100,13 +100,13 @@ func (c *configWatcher) updateWorkload(ctx context.Context, wl, oldWl k8sapi.Wor c.Store(scx) ac := scx.AgentConfig() dlog.Debugf(ctx, "deleting pods with config mismatch for %s %s.%s", ac.WorkloadKind, ac.WorkloadName, ac.Namespace) - err = c.DeletePodsWithConfigMismatch(ctx, scx) + err = c.EvictPodsWithAgentConfigMismatch(ctx, scx) if err != nil { dlog.Error(ctx, err) } case "false", "disabled": c.Delete(wl.GetName(), wl.GetNamespace()) - err := c.DeletePodsWithConfig(ctx, wl) + err := c.EvictPodsWithAgentConfig(ctx, wl) if err != nil { dlog.Error(ctx, err) } diff --git a/cmd/traffic/cmd/manager/state/intercept.go b/cmd/traffic/cmd/manager/state/intercept.go index 8ff1bad087..f2f0b54190 100644 --- a/cmd/traffic/cmd/manager/state/intercept.go +++ b/cmd/traffic/cmd/manager/state/intercept.go @@ -30,6 +30,7 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/agentmap" "github.com/telepresenceio/telepresence/v2/pkg/errcat" + "github.com/telepresenceio/telepresence/v2/pkg/ioutil" "github.com/telepresenceio/telepresence/v2/pkg/k8sapi" "github.com/telepresenceio/telepresence/v2/pkg/tunnel" ) @@ -143,12 +144,12 @@ func (s *state) PrepareIntercept( func prepareAllContainerPorts(cn *agentconfig.Container, pi *rpc.PreparedIntercept) { pics := agentconfig.PortUniqueIntercepts(cn) if ni := len(pics); ni > 0 { - // Put first port in the intercept itself + // Put the first port in the intercept itself i0 := pics[0] pi.ContainerPort = int32(i0.ContainerPort) pi.Protocol = string(i0.Protocol) if ni > 1 { - // Put remaining ports in PodPorts with a 1:1 mapping to target port on client. + // Put the remaining ports in PodPorts with a 1:1 mapping to target port on the client. pi.PodPorts = make([]string, ni-1) for i := 1; i < ni; i++ { ic := pics[i] @@ -292,8 +293,8 @@ func (s *state) AddIntercept(ctx context.Context, cir *rpc.CreateInterceptReques pmSpec.PodPorts = nil pmSpec.LocalPorts = nil - // This intercept targets a pod-port (container port) directly. The name of the container - // is not necessary, because container ports must be unique within the pod. + // This intercept targets a pod-port (container port) directly. A container name + // is not necessary because container ports must be unique within the pod. pmSpec.ServiceUid = "" pmSpec.ServicePortName = "" pmSpec.ServicePort = 0 @@ -331,7 +332,7 @@ func IsChildIntercept(spec *rpc.InterceptSpec) bool { func (s *state) addIntercept(id string, cir *rpc.CreateInterceptRequest) (*Intercept, error) { is := s.self.NewInterceptInfo(id, cir) - // Wrap each potential-state-change in a + // Wrap each potential-state-change in an // // if cept.Disposition == rpc.InterceptDispositionType_WAITING { … } // @@ -470,7 +471,7 @@ func (s *state) ensureAgent(parentCtx context.Context, wl k8sapi.Workload, exten if err != nil { return nil, nil, err } - err = mutator.GetMap(ctx).DeletePodsWithConfigMismatch(ctx, sce) + err = mutator.GetMap(ctx).EvictPodsWithAgentConfigMismatch(ctx, sce) if err != nil { dlog.Errorf(ctx, "failed to inactivate pods: %v", err) return nil, nil, err @@ -534,7 +535,7 @@ func (s *state) restoreAppContainer(ctx context.Context, ii *rpc.InterceptInfo) // The pods for this workload will be killed once the new updated sidecar // reaches the configmap. We inactivate them now, so that they don't continue to // review intercepts. - err = mm.DeletePodsWithConfigMismatch(ctx, sce) + err = mm.EvictPodsWithAgentConfigMismatch(ctx, sce) return sce, err }) return err @@ -701,7 +702,7 @@ func watchFailedInjectionEvents(ctx context.Context, name, namespace string) (<- if !ok { return } - // Using a negated Before when comparing the timestamps here is relevant. They will often be equal and still relevant + // Using negated Before when comparing the timestamps here is relevant. They will often be equal and still relevant if e, ok := eo.Object.(*events.Event); ok && !e.CreationTimestamp.Time.Before(start) && !strings.HasPrefix(e.Note, "(combined from similar events):") { @@ -738,7 +739,7 @@ func (s *state) waitForAgents(ctx context.Context, ac *agentconfig.Sidecar, fail } msg := fe.Note // Terminate directly on known fatal events. No need for the user to wait for a timeout - // when one of these are encountered. + // when one of those is encountered. switch fe.Reason { case "BackOff": // The traffic-agent container was injected, but it fails to start @@ -807,7 +808,7 @@ func (s *state) waitForAgents(ctx context.Context, ac *agentconfig.Sidecar, fail v = "timed out" } bf := &strings.Builder{} - fmt.Fprintf(bf, "request %s while waiting for agent %s.%s to arrive", v, name, namespace) + ioutil.Printf(bf, "request %s while waiting for agent %s.%s to arrive", v, name, namespace) if len(fes) > 0 { bf.WriteString(": Events that may be relevant:\n") writeEventList(bf, fes) @@ -845,9 +846,9 @@ func writeEventList(bf *strings.Builder, es []*events.Event) { typeLen += 3 reasonLen += 3 objectLen += 3 - fmt.Fprintf(bf, "%-*s%-*s%-*s%-*s%s\n", ageLen, "AGE", typeLen, "TYPE", reasonLen, "REASON", objectLen, "OBJECT", "MESSAGE") + ioutil.Printf(bf, "%-*s%-*s%-*s%-*s%s\n", ageLen, "AGE", typeLen, "TYPE", reasonLen, "REASON", objectLen, "OBJECT", "MESSAGE") for _, e := range es { - fmt.Fprintf(bf, "%-*s%-*s%-*s%-*s%s\n", ageLen, age(e), typeLen, e.Type, reasonLen, e.Reason, objectLen, object(e), e.Note) + ioutil.Printf(bf, "%-*s%-*s%-*s%-*s%s\n", ageLen, age(e), typeLen, e.Type, reasonLen, e.Reason, objectLen, object(e), e.Note) } } diff --git a/cmd/traffic/cmd/manager/state/state.go b/cmd/traffic/cmd/manager/state/state.go index 30a7c6ea10..d779f5481d 100644 --- a/cmd/traffic/cmd/manager/state/state.go +++ b/cmd/traffic/cmd/manager/state/state.go @@ -611,7 +611,7 @@ func (s *state) UninstallAgents(ctx context.Context, ur *rpc.UninstallAgentsRequ mm := mutator.GetMap(ctx) agents := ur.Agents if len(agents) == 0 { - if err := mm.DeleteAllPodsWithConfig(ctx, ns); err != nil { + if err := mm.EvictAllPodsWithAgentConfig(ctx, ns); err != nil { return status.Errorf(codes.Internal, "unable to delete pods with agent: %v", err) } return nil @@ -628,7 +628,7 @@ func (s *state) UninstallAgents(ctx context.Context, ur *rpc.UninstallAgentsRequ for _, wl := range wls { mm.Delete(wl.GetName(), ns) - if err := mm.DeletePodsWithConfig(ctx, wl); err != nil { + if err := mm.EvictPodsWithAgentConfig(ctx, wl); err != nil { return status.Errorf(codes.Internal, "unable to delete agent for workload %s.%s: %v", wl.GetName(), ns, err) } } From e9285e0ab6619d5456a7a976438c097ddb6909ee Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Thu, 6 Feb 2025 11:41:28 +0100 Subject: [PATCH 43/61] Safeguard concurrent access to agentpf.client with RWMutex. Signed-off-by: Thomas Hallgren --- .../agent_injector_disabled_test.go | 1 + integration_test/manual_agent_test.go | 1 + pkg/client/agentpf/clients.go | 67 +++++++++++-------- 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/integration_test/agent_injector_disabled_test.go b/integration_test/agent_injector_disabled_test.go index b25b645ec8..e94b9ec434 100644 --- a/integration_test/agent_injector_disabled_test.go +++ b/integration_test/agent_injector_disabled_test.go @@ -38,6 +38,7 @@ func (s *agentInjectorDisabledSuite) Test_AgentInjectorDisabled() { s.ApplyApp(ctx, svc, "deploy/"+svc) defer s.DeleteSvcAndWorkload(ctx, "deploy", svc) + s.CapturePodLogs(ctx, svc, "traffic-agent", s.AppNamespace()) s.TelepresenceConnect(ctx) _, sErr, err := itest.Telepresence(ctx, "intercept", svc) diff --git a/integration_test/manual_agent_test.go b/integration_test/manual_agent_test.go index 1c9ebf4d3b..77cd0a194e 100644 --- a/integration_test/manual_agent_test.go +++ b/integration_test/manual_agent_test.go @@ -102,6 +102,7 @@ func testManualAgent(s *itest.Suite, nsp itest.NamespacePair) { }() err = nsp.RolloutStatusWait(ctx, "deploy/"+ac.WorkloadName) + nsp.CapturePodLogs(ctx, ac.WorkloadName, "traffic-agent", nsp.AppNamespace()) require.NoError(err) nsp.TelepresenceConnect(ctx) diff --git a/pkg/client/agentpf/clients.go b/pkg/client/agentpf/clients.go index 78310c6da8..f19f60ec94 100644 --- a/pkg/client/agentpf/clients.go +++ b/pkg/client/agentpf/clients.go @@ -31,7 +31,7 @@ type client struct { // cancelClient // cancelDialWatch // cli and cancelClient are both safe to use without a mutex once the ready channel is closed. - sync.Mutex + sync.RWMutex cli agent.AgentClient session *manager.SessionInfo info *manager.AgentPodInfo @@ -51,12 +51,10 @@ func (ac *client) String() string { return fmt.Sprintf("%s.%s:%d", ai.PodName, ai.Namespace, ai.ApiPort) } -func (ac *client) ensureConnect(ctx context.Context) error { +func (ac *client) ensureConnect(ctx context.Context) (err error) { ac.Lock() infant := ac.infant - if infant { - ac.infant = false - } + ac.infant = false ac.Unlock() if infant { go ac.connect(ctx, func() { @@ -64,24 +62,27 @@ func (ac *client) ensureConnect(ctx context.Context) error { }) } select { - case err, ok := <-ac.ready: - if ok { - // Put error back on channel in case this Tunnel is used again before it's deleted. - ac.ready <- err - return err - } - // ready channel is closed. We are ready to go. case <-ctx.Done(): - return ctx.Err() + err = ctx.Err() + case err = <-ac.ready: + // Put status on the channel for next call to ensureConnect. + ac.ready <- err } - return nil + return err } func (ac *client) Tunnel(ctx context.Context, opts ...grpc.CallOption) (tunnel.Client, error) { if err := ac.ensureConnect(ctx); err != nil { return nil, err } - tc, err := ac.cli.Tunnel(ctx, opts...) + ac.RLock() + cli := ac.cli + ac.RUnlock() + if cli == nil { + // Client was closed. + return nil, io.EOF + } + tc, err := cli.Tunnel(ctx, opts...) if err != nil { return nil, err } @@ -101,12 +102,10 @@ func (ac *client) connect(ctx context.Context, deleteMe func()) { var err error defer func() { - if err == nil { - close(ac.ready) - } else { + if err != nil { deleteMe() - ac.ready <- err } + ac.ready <- err }() var conn *grpc.ClientConn @@ -127,6 +126,9 @@ func (ac *client) connect(ctx context.Context, deleteMe func()) { ac.cancelClient = nil ac.cli = nil ac.infant = true + for len(ac.ready) > 0 { + <-ac.ready + } ac.Unlock() }() } @@ -138,24 +140,24 @@ func (ac *client) connect(ctx context.Context, deleteMe func()) { } func (ac *client) dormant() bool { - ac.Lock() + ac.RLock() dormant := !(ac.infant || ac.cli == nil || ac.info.Intercepted) && atomic.LoadInt32(&ac.tunnelCount) == 0 - ac.Unlock() + ac.RUnlock() return dormant } func (ac *client) intercepted() bool { - ac.Lock() + ac.RLock() ret := ac.info.Intercepted - ac.Unlock() + ac.RUnlock() return ret } func (ac *client) cancel() bool { - ac.Lock() + ac.RLock() cc := ac.cancelClient cdw := ac.cancelDialWatch - ac.Unlock() + ac.RUnlock() didCancel := false if cc != nil { didCancel = true @@ -169,9 +171,9 @@ func (ac *client) cancel() bool { } func (ac *client) setIntercepted(ctx context.Context, k string, status bool) { - ac.Lock() + ac.RLock() aci := ac.info.Intercepted - ac.Unlock() + ac.RUnlock() if status { if aci { return @@ -188,9 +190,9 @@ func (ac *client) setIntercepted(ctx context.Context, k string, status bool) { // This agent is no longer intercepting. Stop the dial watcher dlog.Debugf(ctx, "Agent %s changed to not intercepted", k) - ac.Lock() + ac.RLock() cdw := ac.cancelDialWatch - ac.Unlock() + ac.RUnlock() if cdw != nil { cdw() } @@ -206,6 +208,12 @@ func (ac *client) startDialWatcher(ctx context.Context) error { } func (ac *client) startDialWatcherReady(ctx context.Context) error { + ac.RLock() + cli := ac.cli + ac.RUnlock() + if cli == nil { + return fmt.Errorf("agent connection closed") + } ctx, cancel := context.WithCancel(ctx) // Create the dial watcher @@ -232,6 +240,7 @@ func (ac *client) startDialWatcherReady(ctx context.Context) error { if err != nil { dlog.Error(ctx, err) } + // The traffic-agent closed the dial watcher which means that it's terminating. ac.cancel() }() return nil From cd33e21b24ba1b1c14cdb3b16b9517fb30f0f7bf Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Thu, 6 Feb 2025 22:31:48 +0100 Subject: [PATCH 44/61] Add better debug logging around the agent pod watcher mechanism. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/manager/service.go | 34 ++++++++++++++++-------------- pkg/client/agentpf/clients.go | 7 ++++++ 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index a2d8469846..009a12192f 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -292,9 +292,9 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa } var interceptInfos map[string]*state.Intercept - isIntercepted := func(name, namespace string) bool { + isIntercepted := func(a *rpc.AgentPodInfo) bool { for _, ii := range interceptInfos { - if name == ii.Spec.Agent && namespace == ii.Spec.Namespace { + if a.WorkloadName == ii.Spec.Agent && a.Namespace == ii.Spec.Namespace { return true } } @@ -319,15 +319,16 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa if err != nil { dlog.Errorf(ctx, "error parsing agent pod ip %q: %v", a.PodIp, err) } - agents[i] = &rpc.AgentPodInfo{ + ap := &rpc.AgentPodInfo{ WorkloadName: a.Name, PodName: a.PodName, Namespace: a.Namespace, PodIp: aip.AsSlice(), ApiPort: a.ApiPort, - Intercepted: isIntercepted(a.Name, a.Namespace), } - agentNames[i] = a.Name + ap.Intercepted = isIntercepted(ap) + agents[i] = ap + agentNames[i] = fmt.Sprintf("%s(%s)", ap.PodName, net.IP(ap.PodIp)) i++ } case is, ok := <-interceptsCh: @@ -335,11 +336,12 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa return nil } interceptInfos = is - for i, a := range agents { - a.Intercepted = isIntercepted(agentNames[i], a.Namespace) + for _, ap := range agents { + ap.Intercepted = isIntercepted(ap) } } if agents != nil { + dlog.Debugf(ctx, "Sending update for %s", agentNames) if err = stream.Send(&rpc.AgentPodInfoSnapshot{Agents: agents}); err != nil { return err } @@ -412,7 +414,7 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(tunnel.Sess case snapshot, ok := <-snapshotCh: if !ok { // The request has been canceled. - dlog.Debug(ctx, "WatchAgentsNS request cancelled") + dlog.Debug(ctx, "Request cancelled") return nil } agentSessionIDs := slices.Sorted(maps.Keys(snapshot)) @@ -433,7 +435,7 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(tunnel.Sess names[i] = a.PodName + "." + a.Namespace i++ } - dlog.Tracef(ctx, "WatchAgentsNS sending update %v", names) + dlog.Tracef(ctx, "Sending update %v", names) } resp := &rpc.AgentInfoSnapshot{ Agents: agents, @@ -443,7 +445,7 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(tunnel.Sess } case <-sessionDone: // Manager believes this session has ended. - dlog.Debug(ctx, "WatchAgentsNS session cancelled") + dlog.Debug(ctx, "Session cancelled") return nil } } @@ -473,7 +475,7 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W return false } if as := s.state.GetAgent(sessionID); as == nil { - dlog.Debugf(ctx, "WatchIntercepts session no longer active") + dlog.Debugf(ctx, "Session no longer active") return false } // Don't return intercepts that aren't in a "agent-owned" state. @@ -505,10 +507,10 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W select { case snapshot, ok := <-snapshotCh: if !ok { - dlog.Debugf(ctx, "WatchIntercepts request cancelled") + dlog.Debugf(ctx, "Request cancelled") return nil } - dlog.Tracef(ctx, "WatchIntercepts sending update") + dlog.Tracef(ctx, "Sending update") intercepts := make([]*rpc.InterceptInfo, 0, len(snapshot)) for _, intercept := range snapshot { intercepts = append(intercepts, intercept.InterceptInfo) @@ -520,14 +522,14 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W return intercepts[i].Id < intercepts[j].Id }) if err := stream.Send(resp); err != nil { - dlog.Debugf(ctx, "WatchIntercepts encountered a write error: %v", err) + dlog.Debugf(ctx, "Encountered a write error: %v", err) return err } case <-ctx.Done(): - dlog.Debugf(ctx, "WatchIntercepts context cancelled") + dlog.Debugf(ctx, "Context cancelled") return nil case <-sessionDone: - dlog.Debugf(ctx, "WatchIntercepts session cancelled") + dlog.Debugf(ctx, "Session cancelled") return nil } } diff --git a/pkg/client/agentpf/clients.go b/pkg/client/agentpf/clients.go index f19f60ec94..2aac863afb 100644 --- a/pkg/client/agentpf/clients.go +++ b/pkg/client/agentpf/clients.go @@ -498,6 +498,13 @@ func (s *clients) WaitForWorkload(ctx context.Context, timeout time.Duration, na func (s *clients) updateClients(ctx context.Context, ais []*manager.AgentPodInfo) error { defer s.notifyWaiters() + if dlog.MaxLogLevel(ctx) >= dlog.LogLevelDebug { + ns := make([]string, len(ais)) + for i, ac := range ais { + ns[i] = fmt.Sprintf("%s(%s)", ac.PodName, net.IP(ac.PodIp)) + } + dlog.Debugf(ctx, "updateClients %s", ns) + } var aim map[string]*manager.AgentPodInfo if len(ais) > 0 { aim = make(map[string]*manager.AgentPodInfo, len(ais)) From 32109c99be3935b8b00ed8c65d003d91659ed66e Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Thu, 6 Feb 2025 22:33:52 +0100 Subject: [PATCH 45/61] Don't let the agent pod watcher wait for port-forward to be established Signed-off-by: Thomas Hallgren --- pkg/client/agentpf/clients.go | 41 +++++++++++++++-------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/pkg/client/agentpf/clients.go b/pkg/client/agentpf/clients.go index 2aac863afb..1893464e12 100644 --- a/pkg/client/agentpf/clients.go +++ b/pkg/client/agentpf/clients.go @@ -40,7 +40,7 @@ type client struct { cancelClient context.CancelFunc cancelDialWatch context.CancelFunc tunnelCount int32 - infant bool + infant atomic.Bool } func (ac *client) String() string { @@ -52,11 +52,7 @@ func (ac *client) String() string { } func (ac *client) ensureConnect(ctx context.Context) (err error) { - ac.Lock() - infant := ac.infant - ac.infant = false - ac.Unlock() - if infant { + if ac.infant.CompareAndSwap(true, false) { go ac.connect(ctx, func() { ac.remove() }) @@ -125,7 +121,7 @@ func (ac *client) connect(ctx context.Context, deleteMe func()) { conn.Close() ac.cancelClient = nil ac.cli = nil - ac.infant = true + ac.infant.Store(true) for len(ac.ready) > 0 { <-ac.ready } @@ -140,8 +136,11 @@ func (ac *client) connect(ctx context.Context, deleteMe func()) { } func (ac *client) dormant() bool { + if ac.infant.Load() || atomic.LoadInt32(&ac.tunnelCount) > 0 { + return false + } ac.RLock() - dormant := !(ac.infant || ac.cli == nil || ac.info.Intercepted) && atomic.LoadInt32(&ac.tunnelCount) == 0 + dormant := ac.cli != nil && !ac.info.Intercepted ac.RUnlock() return dormant } @@ -173,29 +172,23 @@ func (ac *client) cancel() bool { func (ac *client) setIntercepted(ctx context.Context, k string, status bool) { ac.RLock() aci := ac.info.Intercepted + cdw := ac.cancelDialWatch ac.RUnlock() if status { if aci { return } dlog.Debugf(ctx, "Agent %s changed to intercepted", k) - if err := ac.startDialWatcher(ctx); err != nil { - dlog.Errorf(ctx, "failed to start client watcher for %s: %v", k, err) - } + go func() { + if err := ac.startDialWatcher(ctx); err != nil { + dlog.Errorf(ctx, "failed to start client watcher for %s: %v", k, err) + } + }() // This agent is now intercepting. Start a dial watcher. - } else { - if !aci { - return - } - + } else if aci && cdw != nil { // This agent is no longer intercepting. Stop the dial watcher dlog.Debugf(ctx, "Agent %s changed to not intercepted", k) - ac.RLock() - cdw := ac.cancelDialWatch - ac.RUnlock() - if cdw != nil { - cdw() - } + cdw() } } @@ -557,9 +550,9 @@ func (s *clients) updateClients(ctx context.Context, ais []*manager.AgentPodInfo remove: func() { deleteClient(k) }, - info: ai, - infant: true, + info: ai, } + ac.infant.Store(true) dlog.Debugf(ctx, "Adding agent pod %s (%s)", k, net.IP(ai.PodIp)) return ac, false }) From 4bd28ea5fa1b9aafed6da51b314bca52a4066506 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Fri, 7 Feb 2025 18:38:41 +0100 Subject: [PATCH 46/61] Introduce a proper tunnel.Tag to make some sense out of trace logging. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/agent/agent.go | 4 +- cmd/traffic/cmd/agent/client.go | 2 +- cmd/traffic/cmd/agent/fwdstate.go | 7 ++-- cmd/traffic/cmd/agent/server.go | 5 ++- cmd/traffic/cmd/agent/state_test.go | 3 +- cmd/traffic/cmd/manager/service.go | 6 ++- pkg/client/agentpf/clients.go | 2 +- pkg/client/remotefs/bridge.go | 2 +- pkg/client/rootd/dns/server_linux.go | 3 +- pkg/client/rootd/stream_creator.go | 2 +- pkg/client/userd/trafficmgr/dial_request.go | 3 +- pkg/client/userd/trafficmgr/podaccess.go | 3 +- pkg/forwarder/interceptor.go | 12 ++++-- pkg/forwarder/tcp.go | 5 ++- pkg/forwarder/udp.go | 34 +++++++++-------- pkg/tunnel/client_stream.go | 4 +- pkg/tunnel/dialer.go | 41 +++++++++++---------- pkg/tunnel/metrics.go | 7 ++-- pkg/tunnel/pipe.go | 10 ++--- pkg/tunnel/server_stream.go | 4 +- pkg/tunnel/stream.go | 41 ++++++++++++++++----- pkg/tunnel/stream_test.go | 20 +++++----- pkg/tunnel/udplistener.go | 22 ++++++----- 23 files changed, 145 insertions(+), 97 deletions(-) diff --git a/cmd/traffic/cmd/agent/agent.go b/cmd/traffic/cmd/agent/agent.go index 6b904c7e8a..70c30bd774 100644 --- a/cmd/traffic/cmd/agent/agent.go +++ b/cmd/traffic/cmd/agent/agent.go @@ -210,9 +210,9 @@ func sidecar(ctx context.Context, s State, info *rpc.AgentInfo) error { // Redirect non-intercepted traffic to the pod, so that injected sidecars that hijack the ports for // incoming connections will continue to work. targetHost := s.PodIP() - fwd = forwarder.NewInterceptor(pp, targetHost, cp) + fwd = forwarder.NewInterceptor(pp, tunnel.AgentToProxied, targetHost, cp) } else { - fwd = forwarder.NewInterceptor(pp, "", 0) + fwd = forwarder.NewInterceptor(pp, tunnel.AgentToClient, "", 0) cp = ic.ContainerPort } diff --git a/cmd/traffic/cmd/agent/client.go b/cmd/traffic/cmd/agent/client.go index 36df189c00..d3fc3f3a52 100644 --- a/cmd/traffic/cmd/agent/client.go +++ b/cmd/traffic/cmd/agent/client.go @@ -125,7 +125,7 @@ func TalkToManager(ctx context.Context, address string, info *rpc.AgentInfo, sta return err } wg.Go("dialWait", func(ctx context.Context) error { - return tunnel.DialWaitLoop(ctx, tunnel.ManagerProvider(manager), dialerStream, tunnel.SessionID(session.SessionId)) + return tunnel.DialWaitLoop(ctx, tunnel.ManagerToAgent, tunnel.ManagerProvider(manager), dialerStream, tunnel.SessionID(session.SessionId)) }) // Deal with log-level changes diff --git a/cmd/traffic/cmd/agent/fwdstate.go b/cmd/traffic/cmd/agent/fwdstate.go index 2b1ed57db0..31dfb6dd53 100644 --- a/cmd/traffic/cmd/agent/fwdstate.go +++ b/cmd/traffic/cmd/agent/fwdstate.go @@ -63,10 +63,11 @@ func (pm *ProviderMux) ReportMetrics(ctx context.Context, metrics *manager.Tunne pm.AgentProvider.ReportMetrics(ctx, metrics) } -func (pm *ProviderMux) CreateClientStream(ctx context.Context, sessionID tunnel.SessionID, id tunnel.ConnID, roundTripLatency, dialTimeout time.Duration) (tunnel.Stream, error) { - s, err := pm.AgentProvider.CreateClientStream(ctx, sessionID, id, roundTripLatency, dialTimeout) +func (pm *ProviderMux) CreateClientStream(ctx context.Context, tag tunnel.Tag, sessionID tunnel.SessionID, id tunnel.ConnID, roundTripLatency, dialTimeout time.Duration, +) (tunnel.Stream, error) { + s, err := pm.AgentProvider.CreateClientStream(ctx, tag, sessionID, id, roundTripLatency, dialTimeout) if err == nil && s == nil { - s, err = pm.ManagerProvider.CreateClientStream(ctx, sessionID, id, roundTripLatency, dialTimeout) + s, err = pm.ManagerProvider.CreateClientStream(ctx, tag, sessionID, id, roundTripLatency, dialTimeout) } return s, err } diff --git a/cmd/traffic/cmd/agent/server.go b/cmd/traffic/cmd/agent/server.go index d9b1cf2b39..e120671b7f 100644 --- a/cmd/traffic/cmd/agent/server.go +++ b/cmd/traffic/cmd/agent/server.go @@ -27,7 +27,7 @@ func (s *state) Version(context.Context, *emptypb.Empty) (*rpc.VersionInfo2, err func (s *state) Tunnel(server agent.Agent_TunnelServer) error { ctx := server.Context() - stream, err := tunnel.NewServerStream(ctx, server) + stream, err := tunnel.NewServerStream(ctx, tunnel.ClientToAgent, server) if err != nil { return status.Errorf(codes.FailedPrecondition, "failed to connect stream: %v", err) } @@ -80,7 +80,8 @@ func (s *state) WatchDial(session *rpc.SessionInfo, server agent.Agent_WatchDial } } -func (s *state) CreateClientStream(ctx context.Context, sessionID tunnel.SessionID, id tunnel.ConnID, roundTripLatency, dialTimeout time.Duration) (tunnel.Stream, error) { +func (s *state) CreateClientStream(ctx context.Context, _ tunnel.Tag, sessionID tunnel.SessionID, id tunnel.ConnID, roundTripLatency, dialTimeout time.Duration, +) (tunnel.Stream, error) { dlog.Debugf(ctx, "Creating tunnel to client %s for id %s", sessionID, id) drCh, ok := s.dialWatchers.Load(sessionID) var stCh <-chan tunnel.Stream diff --git a/cmd/traffic/cmd/agent/state_test.go b/cmd/traffic/cmd/agent/state_test.go index 1bf24fd975..3a6a325e92 100644 --- a/cmd/traffic/cmd/agent/state_test.go +++ b/cmd/traffic/cmd/agent/state_test.go @@ -15,6 +15,7 @@ import ( "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/agent" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/forwarder" + "github.com/telepresenceio/telepresence/v2/pkg/tunnel" ) const ( @@ -23,7 +24,7 @@ const ( ) func makeFS(t *testing.T, ctx context.Context) (forwarder.Interceptor, agent.State) { - f := forwarder.NewInterceptor(agentconfig.PortAndProto{Proto: core.ProtocolTCP, Port: 1111}, appHost, appPort) + f := forwarder.NewInterceptor(agentconfig.PortAndProto{Proto: core.ProtocolTCP, Port: 1111}, tunnel.AgentToProxied, appHost, appPort) go func() { if err := f.Serve(context.Background(), nil); err != nil { dlog.Error(ctx, err) diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index 009a12192f..bef2d7b907 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -719,10 +719,14 @@ func (s *service) removeExcludedEnvVars(envVars map[string]string) { func (s *service) Tunnel(server rpc.Manager_TunnelServer) error { ctx := server.Context() - stream, err := tunnel.NewServerStream(ctx, server) + stream, err := tunnel.NewServerStream(ctx, tunnel.ClientToManager, server) if err != nil { return status.Errorf(codes.FailedPrecondition, "failed to connect stream: %v", err) } + if a := s.state.GetAgent(stream.SessionID()); a != nil { + // This is actually an AgentToManager tunnel. + stream.SetTag(tunnel.AgentToManager) + } return s.state.Tunnel(ctx, stream) } diff --git a/pkg/client/agentpf/clients.go b/pkg/client/agentpf/clients.go index 1893464e12..3b922951dd 100644 --- a/pkg/client/agentpf/clients.go +++ b/pkg/client/agentpf/clients.go @@ -229,7 +229,7 @@ func (ac *client) startDialWatcherReady(ctx context.Context) error { ac.Unlock() go func() { - err := tunnel.DialWaitLoop(ctx, tunnel.AgentProvider(ac.cli), watcher, tunnel.SessionID(ac.session.SessionId)) + err := tunnel.DialWaitLoop(ctx, tunnel.ClientToAgent, tunnel.AgentProvider(ac.cli), watcher, tunnel.SessionID(ac.session.SessionId)) if err != nil { dlog.Error(ctx, err) } diff --git a/pkg/client/remotefs/bridge.go b/pkg/client/remotefs/bridge.go index dd4739263f..19b68178ff 100644 --- a/pkg/client/remotefs/bridge.go +++ b/pkg/client/remotefs/bridge.go @@ -71,7 +71,7 @@ func (m *bridgeMounter) dispatchToTunnel(ctx context.Context, conn net.Conn, pod tos := client2.GetConfig(ctx).Timeouts() ctx, cancel := context.WithCancel(ctx) - s, err := tunnel.NewClientStream(ctx, ms, id, m.sessionID, tos.PrivateRoundtripLatency, tos.PrivateEndpointDial) + s, err := tunnel.NewClientStream(ctx, tunnel.ClientToFileServer, ms, id, m.sessionID, tos.PrivateRoundtripLatency, tos.PrivateEndpointDial) if err != nil { cancel() return fmt.Errorf("failed to create stream: %v", err) diff --git a/pkg/client/rootd/dns/server_linux.go b/pkg/client/rootd/dns/server_linux.go index 02660e4b33..60ab2573f1 100644 --- a/pkg/client/rootd/dns/server_linux.go +++ b/pkg/client/rootd/dns/server_linux.go @@ -18,6 +18,7 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/forwarder" "github.com/telepresenceio/telepresence/v2/pkg/proc" "github.com/telepresenceio/telepresence/v2/pkg/shellquote" + "github.com/telepresenceio/telepresence/v2/pkg/tunnel" "github.com/telepresenceio/telepresence/v2/pkg/vif" ) @@ -135,7 +136,7 @@ func (s *Server) runOverridingServer(c context.Context, dev vif.Device) error { return nil } go func() { - if err = forwarder.ForwardUDP(c, pc.(*net.UDPConn), dnsResolverAddr); err != nil { + if err = forwarder.ForwardUDP(c, tunnel.ClientToDNS, pc.(*net.UDPConn), dnsResolverAddr); err != nil { dlog.Error(c, err) } }() diff --git a/pkg/client/rootd/stream_creator.go b/pkg/client/rootd/stream_creator.go index 06d9283d43..3a21d31b89 100644 --- a/pkg/client/rootd/stream_creator.go +++ b/pkg/client/rootd/stream_creator.go @@ -129,7 +129,7 @@ func (s *Session) streamCreator(ctx context.Context) tunnel.StreamCreator { tc := client.GetConfig(c).Timeouts() return tunnel.NewClientStream( - c, ct, id, tunnel.SessionID(s.session.SessionId), tc.Get(client.TimeoutRoundtripLatency), tc.Get(client.TimeoutEndpointDial)) + c, tunnel.TunToClient, ct, id, tunnel.SessionID(s.session.SessionId), tc.Get(client.TimeoutRoundtripLatency), tc.Get(client.TimeoutEndpointDial)) } } diff --git a/pkg/client/userd/trafficmgr/dial_request.go b/pkg/client/userd/trafficmgr/dial_request.go index 9d79e0ba16..bbd42fcd0a 100644 --- a/pkg/client/userd/trafficmgr/dial_request.go +++ b/pkg/client/userd/trafficmgr/dial_request.go @@ -16,5 +16,6 @@ func (s *session) _dialRequestWatcher(ctx context.Context) error { if err != nil { return err } - return tunnel.DialWaitLoop(ctx, tunnel.ManagerProvider(s.managerClient), dialerStream, tunnel.SessionID(s.sessionInfo.SessionId)) + return tunnel.DialWaitLoop( + ctx, tunnel.ManagerToClient, tunnel.ManagerProvider(s.managerClient), dialerStream, tunnel.SessionID(s.sessionInfo.SessionId)) } diff --git a/pkg/client/userd/trafficmgr/podaccess.go b/pkg/client/userd/trafficmgr/podaccess.go index 854df6df6c..72071898ca 100644 --- a/pkg/client/userd/trafficmgr/podaccess.go +++ b/pkg/client/userd/trafficmgr/podaccess.go @@ -19,6 +19,7 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/client/remotefs" "github.com/telepresenceio/telepresence/v2/pkg/forwarder" "github.com/telepresenceio/telepresence/v2/pkg/iputil" + "github.com/telepresenceio/telepresence/v2/pkg/tunnel" ) type podAccess struct { @@ -138,7 +139,7 @@ func (pa *podAccess) workerPortForward(ctx context.Context, port string, wg *syn dlog.Errorf(ctx, "malformed extra port %q: %v", port, err) return } - f := forwarder.NewInterceptor(pp, pa.podIP, pp.Port) + f := forwarder.NewInterceptor(pp, tunnel.ClientToAgent, pa.podIP, pp.Port) err = f.Serve(ctx, nil) if err != nil && ctx.Err() == nil { dlog.Errorf(ctx, "port-forwarder failed with %v", err) diff --git a/pkg/forwarder/interceptor.go b/pkg/forwarder/interceptor.go index 63c3b60fca..45feb7310b 100644 --- a/pkg/forwarder/interceptor.go +++ b/pkg/forwarder/interceptor.go @@ -19,6 +19,7 @@ import ( type Interceptor interface { io.Closer + Tag() tunnel.Tag InterceptId() string InterceptInfo() *restapi.InterceptInfo Serve(context.Context, chan<- netip.AddrPort) error @@ -36,6 +37,7 @@ type interceptor struct { tCtx context.Context tCancel context.CancelFunc + tag tunnel.Tag targetHost string targetPort uint16 streamProvider tunnel.ClientStreamProvider @@ -43,12 +45,12 @@ type interceptor struct { intercept *manager.InterceptInfo } -func NewInterceptor(from agentconfig.PortAndProto, targetHost string, targetPort uint16) Interceptor { +func NewInterceptor(from agentconfig.PortAndProto, tag tunnel.Tag, targetHost string, targetPort uint16) Interceptor { switch from.Proto { case core.ProtocolTCP: - return newTCP(from.Port, targetHost, targetPort) + return newTCP(from.Port, tag, targetHost, targetPort) case core.ProtocolUDP: - return newUDP(from.Port, targetHost, targetPort) + return newUDP(from.Port, tag, targetHost, targetPort) default: panic(fmt.Errorf("unsupported protocol %s", from.Proto)) } @@ -125,3 +127,7 @@ func (f *interceptor) SetIntercepting(intercept *manager.InterceptInfo) { f.tCtx, f.tCancel = context.WithCancel(f.lCtx) f.intercept = intercept } + +func (f *interceptor) Tag() tunnel.Tag { + return f.tag +} diff --git a/pkg/forwarder/tcp.go b/pkg/forwarder/tcp.go index fd920a76f5..ab8f77f8a0 100644 --- a/pkg/forwarder/tcp.go +++ b/pkg/forwarder/tcp.go @@ -19,9 +19,10 @@ type tcp struct { interceptor } -func newTCP(listenPort uint16, targetHost string, targetPort uint16) Interceptor { +func newTCP(listenPort uint16, tag tunnel.Tag, targetHost string, targetPort uint16) Interceptor { return &tcp{ interceptor: interceptor{ + tag: tag, listenPort: listenPort, targetHost: targetHost, targetPort: targetPort, @@ -178,7 +179,7 @@ func (f *tcp) rerouteConn(ctx context.Context, conn net.Conn, clientSession tunn f.mu.Lock() sp := f.streamProvider f.mu.Unlock() - s, err := sp.CreateClientStream(ctx, clientSession, id, latency, timeout) + s, err := sp.CreateClientStream(ctx, tunnel.AgentToClient, clientSession, id, latency, timeout) if err != nil { cancel() return err diff --git a/pkg/forwarder/udp.go b/pkg/forwarder/udp.go index 41dbbe6878..2625823b29 100644 --- a/pkg/forwarder/udp.go +++ b/pkg/forwarder/udp.go @@ -17,9 +17,10 @@ type udp struct { interceptor } -func newUDP(listenPort uint16, targetHost string, targetPort uint16) Interceptor { +func newUDP(listenPort uint16, tag tunnel.Tag, targetHost string, targetPort uint16) Interceptor { return &udp{ interceptor: interceptor{ + tag: tag, listenPort: listenPort, targetHost: targetHost, targetPort: targetPort, @@ -100,10 +101,10 @@ func (f *udp) forwardConn(ctx context.Context, conn *net.UDPConn) error { if err != nil { return fmt.Errorf("error on resolve(%s): %w", iputil.JoinHostPort(f.targetHost, f.targetPort), err) } - return ForwardUDP(ctx, conn, targetAddr) + return ForwardUDP(ctx, f.tag, conn, targetAddr) } -func ForwardUDP(ctx context.Context, conn *net.UDPConn, targetAddr *net.UDPAddr) error { +func ForwardUDP(ctx context.Context, tag tunnel.Tag, conn *net.UDPConn, targetAddr *net.UDPAddr) error { targets := tunnel.NewPool() la := conn.LocalAddr() dlog.Infof(ctx, "Forwarding udp from %s to %s", la, targetAddr) @@ -113,7 +114,7 @@ func ForwardUDP(ctx context.Context, conn *net.UDPConn, targetAddr *net.UDPAddr) }() ch := make(chan tunnel.UdpReadResult) - go tunnel.UdpReader(ctx, conn, ch) + go tunnel.UdpReader(ctx, tag, conn, ch) for { select { case <-ctx.Done(): @@ -123,13 +124,14 @@ func ForwardUDP(ctx context.Context, conn *net.UDPConn, targetAddr *net.UDPAddr) return nil } id := tunnel.ConnIDFromUDP(rr.Addr, targetAddr) - dlog.Tracef(ctx, "<- SRC udp %s, len %d", id, len(rr.Payload)) + dlog.Tracef(ctx, "<- %s udp %s, len %d", tag, id, len(rr.Payload)) h, _, err := targets.GetOrCreate(ctx, id, func(ctx context.Context, release func()) (tunnel.Handler, error) { tc, err := net.DialUDP("udp", nil, net.UDPAddrFromAddrPort(id.Destination())) if err != nil { return nil, err } return &udpHandler{ + tag: tag, UDPConn: tc, id: id, replyWith: conn, @@ -144,10 +146,10 @@ func ForwardUDP(ctx context.Context, conn *net.UDPConn, targetAddr *net.UDPAddr) for n := 0; n < pn; { wn, err := uh.Write(rr.Payload[n:]) if err != nil { - dlog.Errorf(ctx, "!! TRG udp %s write: %v", id, err) + dlog.Errorf(ctx, "!> %s udp %s write: %v", tag, id, err) return err } - dlog.Tracef(ctx, "-> TRG udp %s, len %d", id, wn) + dlog.Tracef(ctx, "-> %s udp %s, len %d", tag, id, wn) n += wn } } @@ -158,6 +160,7 @@ type udpHandler struct { *net.UDPConn id tunnel.ConnID replyWith net.PacketConn + tag tunnel.Tag release func() } @@ -171,12 +174,12 @@ func (u *udpHandler) Stop(_ context.Context) { } func (u *udpHandler) Start(ctx context.Context) { - go u.forward(ctx) + go u.forward(ctx, u.tag) } -func (u *udpHandler) forward(ctx context.Context) { +func (u *udpHandler) forward(ctx context.Context, tag tunnel.Tag) { ch := make(chan tunnel.UdpReadResult) - go tunnel.UdpReader(ctx, u, ch) + go tunnel.UdpReader(ctx, tag, u, ch) for { select { case <-ctx.Done(): @@ -185,15 +188,15 @@ func (u *udpHandler) forward(ctx context.Context) { if !ok { return } - dlog.Tracef(ctx, "<- TRG udp %s, len %d", u.id, len(rr.Payload)) + dlog.Tracef(ctx, "<- %s udp %s, len %d", tag, u.id, len(rr.Payload)) pn := len(rr.Payload) for n := 0; n < pn; { wn, err := u.replyWith.WriteTo(rr.Payload[n:], net.UDPAddrFromAddrPort(u.id.Source())) if err != nil { - dlog.Errorf(ctx, "!! SRC udp %s write: %v", u.id, err) + dlog.Errorf(ctx, "!> %s udp %s write: %v", tag, u.id, err) return } - dlog.Tracef(ctx, "-> SRC udp %s, len %d", u.id, wn) + dlog.Tracef(ctx, "-> %s udp %s, len %d", tag, u.id, wn) n += wn } } @@ -205,11 +208,12 @@ func (f *udp) interceptConn(ctx context.Context, conn *net.UDPConn, iCept *manag dest := netip.AddrPortFrom(iputil.Parse(spec.TargetHost), uint16(spec.TargetPort)) dlog.Infof(ctx, "Forwarding udp from %s to %s %s", conn.LocalAddr(), spec.Client, dest) defer dlog.Infof(ctx, "Done forwarding udp from %s to %s %s", conn.LocalAddr(), spec.Client, dest) - d := tunnel.NewUDPListener(conn, net.UDPAddrFromAddrPort(dest), func(ctx context.Context, id tunnel.ConnID) (tunnel.Stream, error) { + d := tunnel.NewUDPListener(conn, tunnel.AgentToClient, net.UDPAddrFromAddrPort(dest), func(ctx context.Context, id tunnel.ConnID) (tunnel.Stream, error) { f.mu.Lock() sp := f.streamProvider f.mu.Unlock() - return sp.CreateClientStream(ctx, tunnel.SessionID(iCept.ClientSession.SessionId), id, time.Duration(spec.RoundtripLatency), time.Duration(spec.DialTimeout)) + return sp.CreateClientStream( + ctx, tunnel.AgentToClient, tunnel.SessionID(iCept.ClientSession.SessionId), id, time.Duration(spec.RoundtripLatency), time.Duration(spec.DialTimeout)) }) d.Start(ctx) <-d.Done() diff --git a/pkg/tunnel/client_stream.go b/pkg/tunnel/client_stream.go index 8599d6deea..567a902aad 100644 --- a/pkg/tunnel/client_stream.go +++ b/pkg/tunnel/client_stream.go @@ -12,8 +12,8 @@ type GRPCClientStream interface { CloseSend() error } -func NewClientStream(ctx context.Context, grpcStream GRPCClientStream, id ConnID, sessionID SessionID, callDelay, dialTimeout time.Duration) (Stream, error) { - s := &clientStream{stream: newStream("CLI", grpcStream)} +func NewClientStream(ctx context.Context, tag Tag, grpcStream GRPCClientStream, id ConnID, sessionID SessionID, callDelay, dialTimeout time.Duration) (Stream, error) { + s := &clientStream{stream: newStream(tag, grpcStream)} s.id = id s.roundtripLatency = callDelay s.dialTimeout = dialTimeout diff --git a/pkg/tunnel/dialer.go b/pkg/tunnel/dialer.go index 5d00c22ebe..9e61a54792 100644 --- a/pkg/tunnel/dialer.go +++ b/pkg/tunnel/dialer.go @@ -112,37 +112,38 @@ func (h *dialer) Start(ctx context.Context) { defer close(h.done) id := h.stream.ID() + tag := h.stream.Tag() switch h.connected { case notConnected: // Set up the idle timer to close and release this handler when it's been idle for a while. h.connected = connecting - dlog.Tracef(ctx, " CONN %s, dialing", id) + dlog.Tracef(ctx, " %s %s, dialing", tag, id) d := net.Dialer{Timeout: h.stream.DialTimeout()} conn, err := d.DialContext(ctx, id.DestinationProtocolString(), id.Destination().String()) if err != nil { - dlog.Errorf(ctx, "!! CONN %s, failed to establish connection: %v", id, err) + dlog.Errorf(ctx, "!> %s %s, failed to establish connection: %v", tag, id, err) if err = h.stream.Send(ctx, NewMessage(DialReject, nil)); err != nil { - dlog.Errorf(ctx, "!! CONN %s, failed to send DialReject: %v", id, err) + dlog.Errorf(ctx, "!> %s %s, failed to send DialReject: %v", tag, id, err) } if err = h.stream.CloseSend(ctx); err != nil { - dlog.Errorf(ctx, "!! CONN %s, stream.CloseSend failed: %v", id, err) + dlog.Errorf(ctx, "!> %s %s, stream.CloseSend failed: %v", tag, id, err) } h.connected = notConnected return } if err = h.stream.Send(ctx, NewMessage(DialOK, nil)); err != nil { _ = conn.Close() - dlog.Errorf(ctx, "!! CONN %s, failed to send DialOK: %v", id, err) + dlog.Errorf(ctx, "!> %s %s, failed to send DialOK: %v", tag, id, err) return } - dlog.Tracef(ctx, " CONN %s, dial answered", id) + dlog.Tracef(ctx, "<- %s %s, dial answered", tag, id) h.conn = conn case connecting: default: - dlog.Errorf(ctx, "!! CONN %s, start called in invalid state", id) + dlog.Errorf(ctx, "!! %s %s, start called in invalid state", tag, id) return } @@ -184,6 +185,7 @@ func (h *dialer) connToStreamLoop(ctx context.Context, wg *sync.WaitGroup) { var endReason string endLevel := dlog.LogLevelTrace id := h.stream.ID() + tag := h.stream.Tag() outgoing := make(chan Message, 50) defer func() { @@ -195,7 +197,7 @@ func (h *dialer) connToStreamLoop(ctx context.Context, wg *sync.WaitGroup) { } } close(outgoing) - dlog.Logf(ctx, endLevel, " CONN %s conn-to-stream loop ended because %s", id, endReason) + dlog.Logf(ctx, endLevel, "<- %s %s conn-to-stream loop ended because %s", tag, id, endReason) wg.Done() }() @@ -203,11 +205,11 @@ func (h *dialer) connToStreamLoop(ctx context.Context, wg *sync.WaitGroup) { WriteLoop(ctx, h.stream, outgoing, wg, h.egressBytesProbe) buf := make([]byte, 0x100000) - dlog.Tracef(ctx, " CONN %s conn-to-stream loop started", id) + dlog.Tracef(ctx, "-> %s %s conn-to-stream loop started", tag, id) for { n, err := h.conn.Read(buf) if n > 0 { - dlog.Tracef(ctx, "<- CONN %s, len %d", id, n) + dlog.Tracef(ctx, "-> %s %s, read len %d from conn", tag, id, n) select { case <-ctx.Done(): endReason = ctx.Err().Error() @@ -251,7 +253,7 @@ func (h *dialer) streamToConnLoop(ctx context.Context, wg *sync.WaitGroup) { defer func() { wg.Done() }() - readLoop(ctx, h, h.ingressBytesProbe) + readLoop(ctx, h.stream.Tag(), h, h.ingressBytesProbe) } func handleControl(ctx context.Context, h streamReader, cm Message) { @@ -273,17 +275,17 @@ func handleControl(ctx context.Context, h streamReader, cm Message) { } } -func readLoop(ctx context.Context, h streamReader, trafficProbe *CounterProbe) { +func readLoop(ctx context.Context, tag Tag, h streamReader, trafficProbe *CounterProbe) { var endReason string endLevel := dlog.LogLevelTrace id := h.getStream().ID() defer func() { h.startDisconnect(ctx, endReason) - dlog.Logf(ctx, endLevel, " CONN %s stream-to-conn loop ended because %s", id, endReason) + dlog.Logf(ctx, endLevel, "<- %s %s stream-to-conn loop ended because %s", tag, id, endReason) }() incoming, errCh := ReadLoop(ctx, h.getStream(), trafficProbe) - dlog.Tracef(ctx, " CONN %s stream-to-conn loop started", id) + dlog.Tracef(ctx, "<- %s %s stream-to-conn loop started", tag, id) for { select { case <-ctx.Done(): @@ -319,7 +321,7 @@ func readLoop(ctx context.Context, h streamReader, trafficProbe *CounterProbe) { endLevel = dlog.LogLevelError return } - dlog.Tracef(ctx, "-> CONN %s, len %d", id, wn) + dlog.Tracef(ctx, "<- %s %s, len %d", tag, id, wn) n += wn } } @@ -331,6 +333,7 @@ func readLoop(ctx context.Context, h streamReader, trafficProbe *CounterProbe) { // the dialStream is closed. func DialWaitLoop( ctx context.Context, + tag Tag, tunnelProvider Provider, dialStream rpc.Manager_WatchDialClient, sessionID SessionID, @@ -341,7 +344,7 @@ func DialWaitLoop( for ctx.Err() == nil { dr, err := dialStream.Recv() if err == nil { - go dialRespond(ctx, tunnelProvider, dr, sessionID) + go dialRespond(ctx, tag, tunnelProvider, dr, sessionID) continue } if ctx.Err() != nil { @@ -359,16 +362,16 @@ func DialWaitLoop( return nil } -func dialRespond(ctx context.Context, tunnelProvider Provider, dr *rpc.DialRequest, sessionID SessionID) { +func dialRespond(ctx context.Context, tag Tag, tunnelProvider Provider, dr *rpc.DialRequest, sessionID SessionID) { id := ConnID(dr.ConnId) ctx, cancel := context.WithCancel(ctx) mt, err := tunnelProvider.Tunnel(ctx) if err != nil { - dlog.Errorf(ctx, "!! CONN %s, call to manager Tunnel failed: %v", id, err) + dlog.Errorf(ctx, "!! %s %s, call to manager Tunnel failed: %v", tag, id, err) cancel() return } - s, err := NewClientStream(ctx, mt, id, sessionID, time.Duration(dr.RoundtripLatency), time.Duration(dr.DialTimeout)) + s, err := NewClientStream(ctx, tag, mt, id, sessionID, time.Duration(dr.RoundtripLatency), time.Duration(dr.DialTimeout)) if err != nil { dlog.Error(ctx, err) cancel() diff --git a/pkg/tunnel/metrics.go b/pkg/tunnel/metrics.go index c1bf62c2b7..b01be36aa2 100644 --- a/pkg/tunnel/metrics.go +++ b/pkg/tunnel/metrics.go @@ -10,11 +10,11 @@ import ( ) type StreamProvider interface { - CreateClientStream(ctx context.Context, clientSessionID SessionID, id ConnID, roundTripLatency, dialTimeout time.Duration) (Stream, error) + CreateClientStream(ctx context.Context, tag Tag, clientSessionID SessionID, id ConnID, roundTripLatency, dialTimeout time.Duration) (Stream, error) } type ClientStreamProvider interface { - CreateClientStream(ctx context.Context, clientSessionID SessionID, id ConnID, roundTripLatency, dialTimeout time.Duration) (Stream, error) + CreateClientStream(ctx context.Context, tag Tag, clientSessionID SessionID, id ConnID, roundTripLatency, dialTimeout time.Duration) (Stream, error) ReportMetrics(ctx context.Context, metrics *manager.TunnelMetrics) } @@ -25,6 +25,7 @@ type TrafficManagerStreamProvider struct { func (sp *TrafficManagerStreamProvider) CreateClientStream( ctx context.Context, + tag Tag, clientSessionID SessionID, id ConnID, roundTripLatency, @@ -36,7 +37,7 @@ func (sp *TrafficManagerStreamProvider) CreateClientStream( return nil, fmt.Errorf("call to manager.Tunnel() failed. Id %s: %v", id, err) } - s, err := NewClientStream(ctx, ms, id, sp.AgentSessionID, roundTripLatency, dialTimeout) + s, err := NewClientStream(ctx, tag, ms, id, sp.AgentSessionID, roundTripLatency, dialTimeout) if err != nil { return nil, err } diff --git a/pkg/tunnel/pipe.go b/pkg/tunnel/pipe.go index 7c98d80c66..a72fcb5641 100644 --- a/pkg/tunnel/pipe.go +++ b/pkg/tunnel/pipe.go @@ -12,13 +12,11 @@ func NewPipe(id ConnID, sessionID SessionID) (Stream, Stream) { in := make(chan Message, 1) return &channelStream{ id: id, - tag: "SND", sid: sessionID, recvCh: in, sendCh: out, }, &channelStream{ id: id, - tag: "RCV", sid: sessionID, recvCh: out, sendCh: in, @@ -27,14 +25,16 @@ func NewPipe(id ConnID, sessionID SessionID) (Stream, Stream) { type channelStream struct { id ConnID - tag string sid SessionID recvCh <-chan Message sendCh chan<- Message } -func (s channelStream) Tag() string { - return s.tag +func (s channelStream) SetTag(_ Tag) { +} + +func (s channelStream) Tag() Tag { + return "AcB" } func (s channelStream) ID() ConnID { diff --git a/pkg/tunnel/server_stream.go b/pkg/tunnel/server_stream.go index d3b538ade2..654a8e12f0 100644 --- a/pkg/tunnel/server_stream.go +++ b/pkg/tunnel/server_stream.go @@ -6,8 +6,8 @@ import ( "fmt" ) -func NewServerStream(ctx context.Context, grpcStream GRPCStream) (Stream, error) { - s := &stream{tag: "SRV", grpcStream: grpcStream, syncRatio: 8, ackWindow: 1} +func NewServerStream(ctx context.Context, tag Tag, grpcStream GRPCStream) (Stream, error) { + s := &stream{tag: tag, grpcStream: grpcStream, syncRatio: 8, ackWindow: 1} m, err := s.Receive(ctx) if err != nil { return nil, fmt.Errorf("failed to read initial StreamInfo message: %w", err) diff --git a/pkg/tunnel/stream.go b/pkg/tunnel/stream.go index a024f8b6b1..8e1477192c 100644 --- a/pkg/tunnel/stream.go +++ b/pkg/tunnel/stream.go @@ -22,7 +22,23 @@ import ( // 1 used MuxTunnel instead of one tunnel per connection. const Version = uint16(2) -type SessionID string +type ( + SessionID string + Tag string +) + +const ( + TunToClient = Tag("T⇄C") + ClientToAgent = Tag("C⇄A") + ClientToDNS = Tag("C⇄D") + AgentToClient = Tag("A⇄C") + AgentToProxied = Tag("A⇄P") + ClientToFileServer = Tag("C⇄F") + ClientToManager = Tag("C⇄M") + ManagerToClient = Tag("M⇄C") + AgentToManager = Tag("A⇄M") + ManagerToAgent = Tag("M⇄A") +) // Endpoint is an endpoint for a Stream such as a Dialer or a bidirectional pipe. type Endpoint interface { @@ -59,7 +75,7 @@ type GRPCStream interface { // // When #6 happens, the Stream will simply close. type Stream interface { - Tag() string + Tag() Tag ID() ConnID Receive(context.Context) (Message, error) Send(context.Context, Message) error @@ -68,6 +84,7 @@ type Stream interface { SessionID() SessionID DialTimeout() time.Duration RoundtripLatency() time.Duration + SetTag(tag Tag) } // StreamCreator is a function that creats a Stream. @@ -78,13 +95,13 @@ type StreamCreator func(context.Context, ConnID) (Stream, error) func ReadLoop(ctx context.Context, s Stream, p *CounterProbe) (<-chan Message, <-chan error) { msgCh := make(chan Message, 50) errCh := make(chan error, 1) // Max one message will be sent on this channel - dlog.Tracef(ctx, " %s %s, ReadLoop starting", s.Tag(), s.ID()) + dlog.Tracef(ctx, "<- %s %s, ReadLoop starting", s.Tag(), s.ID()) go func() { var endReason string defer func() { close(errCh) close(msgCh) - dlog.Tracef(ctx, " %s %s, ReadLoop ended: %s", s.Tag(), s.ID(), endReason) + dlog.Tracef(ctx, "<- %s %s, ReadLoop ended: %s", s.Tag(), s.ID(), endReason) }() for { @@ -118,7 +135,7 @@ func ReadLoop(ctx context.Context, s Stream, p *CounterProbe) (<-chan Message, < default: endReason = err.Error() select { - case errCh <- fmt.Errorf("!! %s %s, read from grpc.ClientStream failed: %w", s.Tag(), s.ID(), err): + case errCh <- fmt.Errorf(" %s %s, WriteLoop starting", s.Tag(), s.ID()) go func() { var endReason string defer func() { dlog.Tracef(ctx, " %s %s, WriteLoop ended: %s", s.Tag(), s.ID(), endReason) if err := s.CloseSend(ctx); err != nil { - dlog.Errorf(ctx, "!! %s %s, Send of closeSend failed: %v", s.Tag(), s.ID(), err) + dlog.Errorf(ctx, "!> %s %s, Send of closeSend failed: %v", s.Tag(), s.ID(), err) } wg.Done() }() @@ -183,20 +200,24 @@ type stream struct { dialTimeout time.Duration roundtripLatency time.Duration sessionID SessionID - tag string + tag Tag syncRatio uint32 // send and check sync after each syncRatio message ackWindow uint32 // maximum permitted difference between sent and received ack peerVersion uint16 } -func newStream(tag string, grpcStream GRPCStream) stream { +func newStream(tag Tag, grpcStream GRPCStream) stream { return stream{tag: tag, grpcStream: grpcStream, syncRatio: 8, ackWindow: 1} } -func (s *stream) Tag() string { +func (s *stream) Tag() Tag { return s.tag } +func (s *stream) SetTag(tag Tag) { + s.tag = tag +} + func (s *stream) ID() ConnID { return s.id } diff --git a/pkg/tunnel/stream_test.go b/pkg/tunnel/stream_test.go index df6a924aa6..9a02063563 100644 --- a/pkg/tunnel/stream_test.go +++ b/pkg/tunnel/stream_test.go @@ -119,7 +119,7 @@ func TestStream_Connect(t *testing.T) { wg.Add(2) go func() { defer wg.Done() - client, err := NewClientStream(ctx, tunnel.clientSide(), id, si, 0, 0) + client, err := NewClientStream(ctx, ClientToManager, tunnel.clientSide(), id, si, 0, 0) require.NoError(t, err) assert.Equal(t, Version, client.PeerVersion()) assert.NoError(t, client.CloseSend(ctx)) @@ -127,7 +127,7 @@ func TestStream_Connect(t *testing.T) { go func() { defer wg.Done() - server, err := NewServerStream(ctx, tunnel.serverSide()) + server, err := NewServerStream(ctx, ManagerToClient, tunnel.serverSide()) require.NoError(t, err) assert.Equal(t, id, server.ID()) assert.Equal(t, Version, server.PeerVersion()) @@ -229,7 +229,7 @@ func TestStream_Xfer(t *testing.T) { wg.Add(2) go func() { defer wg.Done() - if client, err := NewClientStream(ctx, tunnel.clientSide(), id, si, 0, 0); err != nil { + if client, err := NewClientStream(ctx, ClientToManager, tunnel.clientSide(), id, si, 0, 0); err != nil { errs <- err } else { produce(ctx, client, large, errs) @@ -237,7 +237,7 @@ func TestStream_Xfer(t *testing.T) { }() go func() { defer wg.Done() - if server, err := NewServerStream(ctx, tunnel.serverSide()); err != nil { + if server, err := NewServerStream(ctx, ManagerToClient, tunnel.serverSide()); err != nil { errs <- err } else { consume(ctx, server, b, errs) @@ -253,7 +253,7 @@ func TestStream_Xfer(t *testing.T) { wg.Add(2) go func() { defer wg.Done() - if server, err := NewServerStream(ctx, tunnel.serverSide()); err != nil { + if server, err := NewServerStream(ctx, ManagerToClient, tunnel.serverSide()); err != nil { errs <- err } else { produce(ctx, server, large, errs) @@ -261,7 +261,7 @@ func TestStream_Xfer(t *testing.T) { }() go func() { defer wg.Done() - if client, err := NewClientStream(ctx, tunnel.clientSide(), id, si, 0, 0); err != nil { + if client, err := NewClientStream(ctx, ClientToManager, tunnel.clientSide(), id, si, 0, 0); err != nil { errs <- err } else { consume(ctx, client, b, errs) @@ -282,7 +282,7 @@ func TestStream_Xfer(t *testing.T) { wg.Add(5) go func() { defer wg.Done() - if s, err := NewServerStream(ctx, ta.serverSide()); err != nil { + if s, err := NewServerStream(ctx, ManagerToClient, ta.serverSide()); err != nil { errs <- err close(aCh) } else { @@ -291,7 +291,7 @@ func TestStream_Xfer(t *testing.T) { }() go func() { defer wg.Done() - if s, err := NewServerStream(ctx, tb.serverSide()); err != nil { + if s, err := NewServerStream(ctx, ManagerToClient, tb.serverSide()); err != nil { errs <- err close(bCh) } else { @@ -300,7 +300,7 @@ func TestStream_Xfer(t *testing.T) { }() go func() { defer wg.Done() - if server, err := NewClientStream(ctx, ta.clientSide(), id, si, 0, 0); err != nil { + if server, err := NewClientStream(ctx, ClientToManager, ta.clientSide(), id, si, 0, 0); err != nil { errs <- err } else { produce(ctx, server, large, errs) @@ -308,7 +308,7 @@ func TestStream_Xfer(t *testing.T) { }() go func() { defer wg.Done() - if client, err := NewClientStream(ctx, tb.clientSide(), id, si, 0, 0); err != nil { + if client, err := NewClientStream(ctx, ClientToManager, tb.clientSide(), id, si, 0, 0); err != nil { errs <- err } else { consume(ctx, client, b, errs) diff --git a/pkg/tunnel/udplistener.go b/pkg/tunnel/udplistener.go index 90b30bc698..06bf8134b9 100644 --- a/pkg/tunnel/udplistener.go +++ b/pkg/tunnel/udplistener.go @@ -15,6 +15,7 @@ import ( // The dialer takes care of dispatching messages between gRPC and UDP connections. type udpListener struct { TimedHandler + tag Tag conn *net.UDPConn connected int32 done chan struct{} @@ -23,13 +24,14 @@ type udpListener struct { creator func(context.Context, ConnID) (Stream, error) } -func NewUDPListener(conn *net.UDPConn, targetAddr *net.UDPAddr, creator func(context.Context, ConnID) (Stream, error)) Endpoint { +func NewUDPListener(conn *net.UDPConn, tag Tag, targetAddr *net.UDPAddr, creator func(context.Context, ConnID) (Stream, error)) Endpoint { state := notConnected if conn != nil { state = connecting } return &udpListener{ TimedHandler: NewTimedHandler("", udpConnTTL, nil), + tag: tag, conn: conn, connected: state, done: make(chan struct{}), @@ -52,7 +54,7 @@ func (h *udpListener) Start(ctx context.Context) { func (h *udpListener) connToStreamLoop(ctx context.Context) { ch := make(chan UdpReadResult) - go UdpReader(ctx, h.conn, ch) + go UdpReader(ctx, h.tag, h.conn, ch) for atomic.LoadInt32(&h.connected) == connected { select { case <-ctx.Done(): @@ -70,7 +72,7 @@ func (h *udpListener) connToStreamLoop(ctx context.Context) { if err != nil { return nil, err } - dlog.Debugf(ctx, " LIS %s conn-to-stream loop started", id) + dlog.Debugf(ctx, "-> %s %s conn-to-stream loop started", h.tag, id) return &udpStream{ TimedHandler: NewTimedHandler(id, udpConnTTL, release), udpListener: h, @@ -78,14 +80,14 @@ func (h *udpListener) connToStreamLoop(ctx context.Context) { }, nil }) if err != nil { - dlog.Errorf(ctx, "!! MGR udp %s get target: %v", id, err) + dlog.Errorf(ctx, ">! %s udp %s get target: %v", h.tag, id, err) return } ps := target.(*udpStream) - dlog.Tracef(ctx, "-> MGR %s, len %d", id, len(rr.Payload)) + dlog.Tracef(ctx, "-> %s %s, len %d", h.tag, id, len(rr.Payload)) err = ps.stream.Send(ctx, NewMessage(Normal, rr.Payload)) if err != nil { - dlog.Errorf(ctx, "!! MGR udp %s write: %v", id, err) + dlog.Errorf(ctx, ">! %s udp %s write: %v", h.tag, id, err) return } } @@ -110,7 +112,7 @@ func (p *udpStream) reply(data []byte) (int, error) { return p.conn.WriteTo(data, net.UDPAddrFromAddrPort(p.ID.Source())) } -func (p *udpStream) startDisconnect(ctx context.Context, s string) { +func (p *udpStream) startDisconnect(context.Context, string) { } func (p *udpStream) Stop(ctx context.Context) { @@ -119,7 +121,7 @@ func (p *udpStream) Stop(ctx context.Context) { func (p *udpStream) Start(ctx context.Context) { p.TimedHandler.Start(ctx) - go readLoop(ctx, p, nil) + go readLoop(ctx, p.stream.Tag(), p, nil) } type UdpReadResult struct { @@ -136,12 +138,12 @@ func IsTimeout(err error) bool { // UdpReader continuously reads from a net.PacketConn and writes the resulting payload and // reply address to a channel. The loop is cancelled when the connection is closed or when // the context is done, at which time the channel is closed. -func UdpReader(ctx context.Context, conn net.PacketConn, ch chan<- UdpReadResult) { +func UdpReader(ctx context.Context, tag Tag, conn net.PacketConn, ch chan<- UdpReadResult) { defer close(ch) var endReason string endLevel := dlog.LogLevelTrace defer func() { - dlog.Logf(ctx, endLevel, " LIS %s UDP read loop ended because %s", conn.LocalAddr(), endReason) + dlog.Logf(ctx, endLevel, "<- %s %s UDP read loop ended because %s", tag, conn.LocalAddr(), endReason) }() buf := [0x10000]byte{} for { From 935b0c8993cc53a9ff2cd1ee752155063342a3dd Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Fri, 7 Feb 2025 18:49:25 +0100 Subject: [PATCH 47/61] Ensure that tunnel write-delays cause connection back-pressure. When reading from a connection and then writing the data on a tunnel, it is essential that any delay when writing also causes the next read from the connection to be delayed. Delaying the connection read causes a back-pressure on the TCP stack, and will make the sender aware that things need to slow down, which is exactly what we want. Closes #3766 Signed-off-by: Thomas Hallgren --- pkg/tunnel/dialer.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/tunnel/dialer.go b/pkg/tunnel/dialer.go index 9e61a54792..2031e699b5 100644 --- a/pkg/tunnel/dialer.go +++ b/pkg/tunnel/dialer.go @@ -187,7 +187,10 @@ func (h *dialer) connToStreamLoop(ctx context.Context, wg *sync.WaitGroup) { id := h.stream.ID() tag := h.stream.Tag() - outgoing := make(chan Message, 50) + // Outgoing must not be buffered. It's essential that messages that are read from the connection + // are sent on the stream a.s.a.p. and that a delay when doing that causes back-pressure on the + // connection. + outgoing := make(chan Message) defer func() { if !h.ResetIdle() { // Hard close of peer. We don't want any more data @@ -204,7 +207,7 @@ func (h *dialer) connToStreamLoop(ctx context.Context, wg *sync.WaitGroup) { wg.Add(1) WriteLoop(ctx, h.stream, outgoing, wg, h.egressBytesProbe) - buf := make([]byte, 0x100000) + buf := make([]byte, 0x80000) dlog.Tracef(ctx, "-> %s %s conn-to-stream loop started", tag, id) for { n, err := h.conn.Read(buf) @@ -224,13 +227,13 @@ func (h *dialer) connToStreamLoop(ctx context.Context, wg *sync.WaitGroup) { endReason = "EOF was encountered" case errors.Is(err, net.ErrClosed): endReason = "the connection was closed" - h.startDisconnect(ctx, endReason) case strings.Contains(err.Error(), "connection aborted"): endReason = "the connection was aborted" default: endReason = fmt.Sprintf("a read error occurred: %T %v", err, err) endLevel = dlog.LogLevelError } + h.startDisconnect(ctx, endReason) return } From 1516e006f281f1b06012eee79524919fa6e6ac7a Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Fri, 7 Feb 2025 22:16:57 +0100 Subject: [PATCH 48/61] Use the default TCP window and 1024 maxInFlight. Signed-off-by: Thomas Hallgren --- pkg/vif/stack.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/vif/stack.go b/pkg/vif/stack.go index de60ae55ee..c04494e240 100644 --- a/pkg/vif/stack.go +++ b/pkg/vif/stack.go @@ -48,13 +48,8 @@ func NewStack(ctx context.Context, dev stack.LinkEndpoint, streamCreator tunnel. return s, nil } -const ( - myWindowScale = 6 - maxReceiveWindow = 1 << (myWindowScale + 14) // 1MiB -) - // maxInFlight specifies the max number of in-flight connection attempts. -const maxInFlight = 512 +const maxInFlight = 1024 // keepAliveIdle is used as the very first alive interval. Subsequent intervals // use keepAliveInterval. @@ -181,7 +176,7 @@ func setTCPHandler(ctx context.Context, s *stack.Stack, streamCreator tunnel.Str mo := tcpip.TCPModerateReceiveBufferOption(true) s.SetTransportProtocolOption(tcp.ProtocolNumber, &mo) - f := tcp.NewForwarder(s, maxReceiveWindow, maxInFlight, func(fr *tcp.ForwarderRequest) { + f := tcp.NewForwarder(s, 0, maxInFlight, func(fr *tcp.ForwarderRequest) { forwardTCP(ctx, streamCreator, fr) }) s.SetTransportProtocolHandler(tcp.ProtocolNumber, f.HandlePacket) From 234803f7445207942207179aab11abc11c9da63d Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Fri, 7 Feb 2025 22:17:46 +0100 Subject: [PATCH 49/61] Print stack-traces on debug-level if session cancellation hangs. Signed-off-by: Thomas Hallgren --- pkg/client/rootd/service.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/client/rootd/service.go b/pkg/client/rootd/service.go index 6617830307..bff4af83c4 100644 --- a/pkg/client/rootd/service.go +++ b/pkg/client/rootd/service.go @@ -6,6 +6,7 @@ import ( "net" "os" "path/filepath" + "runtime" "sync" "sync/atomic" "time" @@ -340,11 +341,15 @@ func (s *Service) startSession(parentCtx context.Context, oi *rpc.NetworkConfig, s.sessionContext = ctx s.sessionCancel = func() { cancel() - wCtx, wCancel := context.WithTimeout(context.Background(), 10*time.Second) - defer wCancel() select { case <-session.Done(): - case <-wCtx.Done(): + case <-time.After(5 * time.Second): + // Something is wrong. The session doesn't die. + if dlog.MaxLogLevel(ctx) >= dlog.LogLevelDebug { + buf := make([]byte, 1024*1024) + n := runtime.Stack(buf, true) + dlog.Debug(ctx, string(buf[:n])) + } } } _ = client.ReloadDaemonLogLevel(ctx, true) From 11e056b9431c759124b11f4a26226cc2094a8272 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Fri, 7 Feb 2025 22:18:12 +0100 Subject: [PATCH 50/61] Slightly more efficient echo-server. Signed-off-by: Thomas Hallgren --- integration_test/testdata/echo-server/main.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/integration_test/testdata/echo-server/main.go b/integration_test/testdata/echo-server/main.go index 6105454598..6fc66a269f 100644 --- a/integration_test/testdata/echo-server/main.go +++ b/integration_test/testdata/echo-server/main.go @@ -320,12 +320,5 @@ func writeRequest(w io.Writer, req *http.Request) { fmt.Fprintf(w, "%s: %s\n", key, value) } } - - var body bytes.Buffer - io.Copy(&body, req.Body) // nolint:errcheck - - if body.Len() > 0 { - fmt.Fprintln(w, "") - body.WriteTo(w) // nolint:errcheck - } + io.Copy(w, req.Body) // nolint:errcheck } From 76d10e0a01a07c0d6dc6b6cd5f18ed68c03384ca Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Fri, 7 Feb 2025 22:19:03 +0100 Subject: [PATCH 51/61] Really massage concurrency in Test_LargeRequest. Signed-off-by: Thomas Hallgren --- integration_test/multiple_services_test.go | 97 ++++++++++++---------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/integration_test/multiple_services_test.go b/integration_test/multiple_services_test.go index 3e635a730e..7399bc5719 100644 --- a/integration_test/multiple_services_test.go +++ b/integration_test/multiple_services_test.go @@ -32,9 +32,9 @@ func init() { } func (s *multipleServicesSuite) Test_LargeRequest() { - const sendSize = 1024 * 1024 * 20 - const varyMax = 1 << 15 // vary last 64Ki - const concurrentRequests = 13 + const sendSize = 1024 * 1024 * 16 + const varyMax = 1024 * 1024 * 4 // vary last 4Mi + const concurrentRequests = 100 tb := [sendSize + varyMax]byte{} tb[0] = '!' @@ -46,48 +46,55 @@ func (s *multipleServicesSuite) Test_LargeRequest() { time.Sleep(3 * time.Second) wg := sync.WaitGroup{} wg.Add(concurrentRequests) - for i := 0; i < concurrentRequests; i++ { - go func(x int) { - defer wg.Done() - sendSize := sendSize + rand.Int()%varyMax // vary the last 64Ki to get random buffer sizes - b := tb[:sendSize] - - // Distribute the requests over all services - url := fmt.Sprintf("http://%s-%d.%s/put", s.Name(), x%s.ServiceCount(), s.AppNamespace()) - req, err := http.NewRequest(http.MethodPut, url, bytes.NewBuffer(b)) - if !s.NoError(err) { - return - } - - client := &http.Client{Timeout: 60 * time.Second} - resp, err := client.Do(req) - if !s.NoError(err) { - return - } - bdy := resp.Body - defer bdy.Close() - if !s.Equal(resp.StatusCode, 200) { - return - } - - cl := sendSize + 1024 - buf := make([]byte, cl) - i := 0 - for i < cl && err == nil { - var j int - j, err = bdy.Read(buf[i:]) - i += j - } - if errors.Is(err, io.EOF) { - err = nil - } - if s.NoError(err) { - ei := bytes.Index(buf, []byte{'!', '\n'}) - s.GreaterOrEqual(ei, 0) - // Do this instead of require.Equal(b, buf[ei:i]) so that on failure we don't print two very large buffers to the terminal - s.Equal(true, bytes.Equal(b, buf[ei:i])) - } - }(i) + pingPong := func(x int) { + defer wg.Done() + sendSize := sendSize + rand.Int()%varyMax // vary the last 64Ki to get random buffer sizes + b := tb[:sendSize] + + // Distribute the requests over all services + url := fmt.Sprintf("http://%s-%d.%s/put", s.Name(), x%s.ServiceCount(), s.AppNamespace()) + req, err := http.NewRequest(http.MethodPut, url, bytes.NewBuffer(b)) + req.ContentLength = int64(len(b)) + if !s.NoError(err) { + return + } + + client := &http.Client{Timeout: 60 * time.Second} + resp, err := client.Do(req) + if !s.NoError(err) { + return + } + bdy := resp.Body + defer bdy.Close() + if !s.Equal(resp.StatusCode, 200) { + return + } + + cl := sendSize + 1024 + buf := make([]byte, cl) + i := 0 + for i < cl && err == nil { + var j int + j, err = bdy.Read(buf[i:]) + i += j + } + if errors.Is(err, io.EOF) { + err = nil + } + if s.NoError(err) { + ei := bytes.Index(buf, []byte{'!', '\n'}) + s.GreaterOrEqual(ei, 0) + // Do this instead of require.Equal(b, buf[ei:i]) so that on failure we don't print two very large buffers to the terminal + s.Equal(true, bytes.Equal(b, buf[ei:i])) + } + } + for i := 0; i < concurrentRequests; i += 4 { + go func() { + pingPong(i) + pingPong(i + 1) + pingPong(i + 2) + pingPong(i + 3) + }() } wg.Wait() } From f34dde3f1d72f4bfaf5cd37eeb19ba0939cadcda Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 8 Feb 2025 12:57:30 +0100 Subject: [PATCH 52/61] Ensure removal of PV and PVC in integration tests. Signed-off-by: Thomas Hallgren --- integration_test/itest/cluster.go | 2 +- integration_test/large_files_test.go | 11 +++++++++-- integration_test/mounts_test.go | 14 ++++++++++---- integration_test/multiple_services_test.go | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/integration_test/itest/cluster.go b/integration_test/itest/cluster.go index 603cad29d6..7b57951082 100644 --- a/integration_test/itest/cluster.go +++ b/integration_test/itest/cluster.go @@ -377,7 +377,7 @@ func (s *cluster) withBasicConfig(c context.Context, t *testing.T) context.Conte config.Grpc().MaxReceiveSizeV, _ = resource.ParseQuantity("10Mi") config.Intercept().UseFtp = true - config.Routing().RecursionBlockDuration = 5 * time.Millisecond + config.Routing().RecursionBlockDuration = 2 * time.Millisecond configYaml, err := config.MarshalYAML() require.NoError(t, err) diff --git a/integration_test/large_files_test.go b/integration_test/large_files_test.go index 64573bcd5e..2c5a981a13 100644 --- a/integration_test/large_files_test.go +++ b/integration_test/large_files_test.go @@ -2,6 +2,7 @@ package integration_test import ( "bufio" + "bytes" "context" "encoding/binary" "errors" @@ -27,6 +28,7 @@ type largeFilesSuite struct { itest.Suite itest.TrafficManager name string + manifest []byte serviceCount int mountPoint []string largeFiles []string @@ -78,18 +80,23 @@ func (s *largeFilesSuite) SetupSuite() { go func(i int) { defer wg.Done() svc := fmt.Sprintf("%s-%d", s.Name(), i) - rdr, err := itest.OpenTemplate(ctx, filepath.Join(k8s, "hello-pv-volume.goyaml"), &itest.PersistentVolume{ + mf, err := itest.ReadTemplate(ctx, filepath.Join(k8s, "hello-pv-volume.goyaml"), &itest.PersistentVolume{ Name: svc, MountDirectory: "/home/scratch", }) + s.manifest = mf require.NoError(err) - require.NoError(s.Kubectl(dos.WithStdin(ctx, rdr), "apply", "-f", "-")) + require.NoError(s.Kubectl(dos.WithStdin(ctx, bytes.NewReader(s.manifest)), "apply", "-f", "-")) s.NoError(itest.RolloutStatusWait(ctx, s.AppNamespace(), "deploy/"+svc)) }(i) } wg.Wait() } +func (s *largeFilesSuite) TeardownSuite() { + s.NoError(s.Kubectl(dos.WithStdin(s.Context(), bytes.NewReader(s.manifest)), "delete", "-f", "-")) +} + func (s *largeFilesSuite) createIntercepts(ctx context.Context) { s.TelepresenceConnect(ctx) diff --git a/integration_test/mounts_test.go b/integration_test/mounts_test.go index 6af336637b..e9257c82f0 100644 --- a/integration_test/mounts_test.go +++ b/integration_test/mounts_test.go @@ -1,6 +1,7 @@ package integration_test import ( + "bytes" "errors" "fmt" "os" @@ -14,6 +15,7 @@ import ( "github.com/datawire/dlib/dlog" "github.com/datawire/dlib/dtime" "github.com/telepresenceio/telepresence/v2/integration_test/itest" + "github.com/telepresenceio/telepresence/v2/pkg/dos" ) type mountsSuite struct { @@ -106,18 +108,22 @@ func (s *mountsSuite) Test_MountWrite() { ctx := s.Context() k8s := filepath.Join("testdata", "k8s") - s.ApplyTemplate(ctx, filepath.Join(k8s, "hello-pv-volume.goyaml"), &itest.PersistentVolume{ + rq := s.Require() + mf, err := itest.ReadTemplate(ctx, filepath.Join(k8s, "hello-pv-volume.goyaml"), &itest.PersistentVolume{ Name: "hello", MountDirectory: "/data", }) - defer s.DeleteSvcAndWorkload(ctx, "deploy", "hello") + rq.NoError(err) + + rq.NoError(s.Kubectl(dos.WithStdin(ctx, bytes.NewReader(mf)), "apply", "-f", "-")) + defer func() { + rq.NoError(s.Kubectl(dos.WithStdin(ctx, bytes.NewReader(mf)), "delete", "-f", "-")) + }() mountPoint := filepath.Join(s.T().TempDir(), "mnt") itest.TelepresenceOk(ctx, "intercept", "hello", "--mount", mountPoint, "--port", "80:80") time.Sleep(2 * time.Second) - rq := s.Require() - content := "hello world\n" path := filepath.Join(mountPoint, "data", "hello.txt") rq.NoError(os.WriteFile(path, []byte(content), 0o644)) diff --git a/integration_test/multiple_services_test.go b/integration_test/multiple_services_test.go index 7399bc5719..5f7d58354e 100644 --- a/integration_test/multiple_services_test.go +++ b/integration_test/multiple_services_test.go @@ -14,6 +14,7 @@ import ( "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/integration_test/itest" + "github.com/telepresenceio/telepresence/v2/pkg/client" ) type multipleServicesSuite struct { @@ -32,6 +33,19 @@ func init() { } func (s *multipleServicesSuite) Test_LargeRequest() { + // This particular cannot run with recursion detection, because it will trigger on the very high concurrency. + ctx := s.Context() + itest.TelepresenceQuitOk(ctx) + ctx = itest.WithConfig(ctx, func(config client.Config) { + config.Routing().RecursionBlockDuration = 0 + }) + s.TelepresenceConnect(ctx) + defer func() { + // Restore the connection to what it was before the test. + itest.TelepresenceQuitOk(ctx) + s.TelepresenceConnect(s.Context()) + }() + const sendSize = 1024 * 1024 * 16 const varyMax = 1024 * 1024 * 4 // vary last 4Mi const concurrentRequests = 100 From 439316215a620c6691b9f0d9104c2dbc3050b0eb Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 8 Feb 2025 14:07:19 +0100 Subject: [PATCH 53/61] Close root daemon connection to traffic-manager early on disconnect. Signed-off-by: Thomas Hallgren --- pkg/client/agentpf/clients.go | 1 + pkg/client/rootd/session.go | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/client/agentpf/clients.go b/pkg/client/agentpf/clients.go index 3b922951dd..ba1a1dfde9 100644 --- a/pkg/client/agentpf/clients.go +++ b/pkg/client/agentpf/clients.go @@ -378,6 +378,7 @@ outer: for ctx.Err() == nil { ais, err := as.Recv() if errors.Is(err, io.EOF) { + // User daemon departed from the session. return nil } switch status.Code(err) { diff --git a/pkg/client/rootd/session.go b/pkg/client/rootd/session.go index 0db090e679..42e923f24c 100644 --- a/pkg/client/rootd/session.go +++ b/pkg/client/rootd/session.go @@ -1025,9 +1025,6 @@ func (s *Session) checkPodConnectivity(ctx context.Context, info *manager.Cluste func (s *Session) run(c context.Context, initErrs chan error) error { defer func() { dlog.Info(c, "-- Session ended") - if s.clientConn != nil { - _ = s.clientConn.Close() - } close(s.done) }() @@ -1129,6 +1126,11 @@ func (s *Session) stop(c context.Context) { // Session already stopped (or is stopping) return } + if s.clientConn != nil { + dlog.Debug(c, "Closing port-forward to traffic-manager") + _ = s.clientConn.Close() + } + dlog.Debug(c, "Bringing down TUN-device") scout.Report(c, "incluster_dns_queries", From 096f34a344c52656b0c62960edabf7fcdf4ece20 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 8 Feb 2025 14:09:07 +0100 Subject: [PATCH 54/61] Let conn<->stream dispatcher close reader and writer of conn separately Signed-off-by: Thomas Hallgren --- pkg/tunnel/dialer.go | 62 ++++++++++++++++++++++++++++++++------- pkg/tunnel/udplistener.go | 2 +- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/pkg/tunnel/dialer.go b/pkg/tunnel/dialer.go index 2031e699b5..664d4c2343 100644 --- a/pkg/tunnel/dialer.go +++ b/pkg/tunnel/dialer.go @@ -30,8 +30,23 @@ const ( notConnected = int32(iota) connecting connected + readClosed + writeClosed ) +type halfReadCloser interface { + CloseRead() error +} + +type halfWriteCloser interface { + CloseWrite() error +} + +type halfCloser interface { + halfReadCloser + halfWriteCloser +} + // streamReader is implemented by the dialer and udpListener so that they can share the // readLoop function. type streamReader interface { @@ -40,7 +55,7 @@ type streamReader interface { Stop(context.Context) getStream() Stream reply([]byte) (int, error) - startDisconnect(context.Context, string) + startDisconnect(context.Context, string, bool) } // The dialer takes care of dispatching messages between gRPC and UDP connections. @@ -166,18 +181,45 @@ func (h *dialer) Done() <-chan struct{} { // Stop will close the underlying TCP/UDP connection. func (h *dialer) Stop(ctx context.Context) { - h.startDisconnect(ctx, "explicit close") + h.startDisconnect(ctx, "explicit close", true) + h.startDisconnect(ctx, "explicit close", false) h.cancel() } -func (h *dialer) startDisconnect(ctx context.Context, reason string) { - if !atomic.CompareAndSwapInt32(&h.connected, connected, notConnected) { +func (h *dialer) startDisconnect(ctx context.Context, reason string, isReader bool) { + if wrConn, ok := h.conn.(halfCloser); ok { + if isReader { + h.startReaderDisconnect(ctx, reason, wrConn) + } else { + h.startWriterDisconnect(ctx, reason, wrConn) + } + } else if atomic.CompareAndSwapInt32(&h.connected, connected, notConnected) { + dlog.Tracef(ctx, "<> %s %s closing connection: %s", h.stream.Tag(), h.stream.ID(), reason) + if err := h.conn.Close(); err != nil { + dlog.Tracef(ctx, "!! %s %s, Close failed: %v", h.stream.Tag(), h.stream.ID(), err) + } + } +} + +func (h *dialer) startReaderDisconnect(ctx context.Context, reason string, conn halfCloser) { + if atomic.CompareAndSwapInt32(&h.connected, connected, readClosed) || atomic.CompareAndSwapInt32(&h.connected, writeClosed, notConnected) { + dlog.Tracef(ctx, "<- %s %s closing connection write: %s", h.stream.Tag(), h.stream.ID(), reason) + if err := conn.CloseWrite(); err != nil { + dlog.Debugf(ctx, " %s %s closing connection read: %s", h.stream.Tag(), h.stream.ID(), reason) + err := conn.CloseRead() + switch { + case err == nil, err == io.EOF, strings.Contains(err.Error(), "not connected"): + default: + dlog.Debugf(ctx, "!< %s %s, CloseRead failed: %v", h.stream.Tag(), h.stream.ID(), err) + } } } @@ -233,7 +275,7 @@ func (h *dialer) connToStreamLoop(ctx context.Context, wg *sync.WaitGroup) { endReason = fmt.Sprintf("a read error occurred: %T %v", err, err) endLevel = dlog.LogLevelError } - h.startDisconnect(ctx, endReason) + h.startDisconnect(ctx, endReason, false) return } @@ -283,7 +325,7 @@ func readLoop(ctx context.Context, tag Tag, h streamReader, trafficProbe *Counte endLevel := dlog.LogLevelTrace id := h.getStream().ID() defer func() { - h.startDisconnect(ctx, endReason) + h.startDisconnect(ctx, endReason, true) dlog.Logf(ctx, endLevel, "<- %s %s stream-to-conn loop ended because %s", tag, id, endReason) }() diff --git a/pkg/tunnel/udplistener.go b/pkg/tunnel/udplistener.go index 06bf8134b9..0c0a205550 100644 --- a/pkg/tunnel/udplistener.go +++ b/pkg/tunnel/udplistener.go @@ -112,7 +112,7 @@ func (p *udpStream) reply(data []byte) (int, error) { return p.conn.WriteTo(data, net.UDPAddrFromAddrPort(p.ID.Source())) } -func (p *udpStream) startDisconnect(context.Context, string) { +func (p *udpStream) startDisconnect(context.Context, string, bool) { } func (p *udpStream) Stop(ctx context.Context) { From 311de3ef677b8ed0c44c593bb746c4f80b2511d2 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 8 Feb 2025 15:57:04 +0100 Subject: [PATCH 55/61] Documentation changes for the new replace command. Introducing `telepresence replace` as an alternative to the `intercept` resulted in a lot of documentation changes. For many (most) use-cases, a `replace` is more natural than an `intercept` unless there's an ability to do personal intercepts, and the OSS edition doesn't provide that. Signed-off-by: Thomas Hallgren --- docs/README.md | 2 +- docs/doc-links.yml | 4 +- docs/faqs.md | 20 +- docs/howtos/cluster-in-vm.md | 2 +- docs/{concepts => howtos}/docker.md | 68 ++++--- docs/howtos/intercepts.md | 270 ++++++++++++++----------- docs/howtos/large-clusters.md | 2 +- docs/quick-start.md | 92 +-------- docs/reference/architecture.md | 12 +- docs/reference/client.md | 8 +- docs/reference/cluster-config.md | 4 +- docs/reference/config.md | 6 +- docs/reference/dns.md | 42 +--- docs/reference/docker-run.md | 40 ++-- docs/reference/environment.md | 20 +- docs/reference/inside-container.md | 6 +- docs/reference/intercepts/cli.md | 16 +- docs/reference/intercepts/container.md | 11 +- docs/reference/intercepts/sidecar.md | 24 ++- docs/reference/rbac.md | 2 +- docs/reference/routing.md | 6 +- docs/reference/tun-device.md | 4 +- docs/reference/volume.md | 15 +- docs/reference/vpn.md | 2 +- docs/troubleshooting.md | 4 +- docs/variables.yml | 4 +- 26 files changed, 310 insertions(+), 376 deletions(-) rename docs/{concepts => howtos}/docker.md (52%) diff --git a/docs/README.md b/docs/README.md index 5c65fe0f8c..0226e5bb37 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,7 +13,7 @@ raw markdown version, more bells and whistles at [telepresence.io](https://telep - Core concepts - [The developer experience and the inner dev loop](concepts/devloop.md) - [Making the remote local: Faster feedback, collaboration and debugging](concepts/faster.md) - - [Using Telepresence with Docker](concepts/docker.md) + - [Using Telepresence with Docker](howtos/docker.md) - [Intercepts](concepts/intercepts.md) - How do I... - [Code and debug an application locally](howtos/intercepts.md) diff --git a/docs/doc-links.yml b/docs/doc-links.yml index febe49ed32..f43ebff81b 100644 --- a/docs/doc-links.yml +++ b/docs/doc-links.yml @@ -16,14 +16,14 @@ link: concepts/devloop - title: "Making the remote local: Faster feedback, collaboration and debugging" link: concepts/faster - - title: Using Telepresence with Docker - link: concepts/docker - title: Intercepts link: concepts/intercepts - title: How do I... items: - title: Code and debug an application locally link: howtos/intercepts + - title: Using Telepresence with Docker + link: howtos/docker - title: Work with large clusters link: howtos/large-clusters - title: Host a cluster in Docker or a VM diff --git a/docs/faqs.md b/docs/faqs.md index 7c75d8a26c..68710556d9 100644 --- a/docs/faqs.md +++ b/docs/faqs.md @@ -14,9 +14,13 @@ Telepresence enables you to connect your local development machine seamlessly to Ultimately, this empowers you to develop services locally and still test integrations with dependent services or data stores running in the remote cluster. -You can “intercept” any requests made to a target Kubernetes workload, and code and debug your associated service locally using your favourite local IDE and in-process debugger. You can test your integrations by making requests against the remote cluster’s ingress and watching how the resulting internal traffic is handled by your service running locally. +Telepresence provides three different ways for you to code, debug, and test your service locally using your favourite local IDE and in-process debugger. -You can also "ingest" a target Kubernetes workload. Very similar to an intercept in that your local workstation has access to the workload's network, environment, and volumes, but no traffic will be rerouted from the cluster +First off, you can "replace" the service with your own local version. This means even though you run your service locally, you can see how it interacts with the rest of the services in the cluster. It's like swapping out a piece of a puzzle and seeing how the whole picture changes. Your local process will have access to the same network, environment, and volumes as the service that it replaces. + +You can also "intercept" any requests made to a service. This is similar to replacing the service, but the remote service will keep running, perform background tasks, and handle traffic that isn't intercepted. + +Finally, you can "ingest" a service. Again, similar to a "replace", but nothing changes in the cluster during an "ingest", and no traffic is routed to the workstation. ** What operating systems does Telepresence work on?** @@ -26,11 +30,11 @@ Telepresence currently works natively on macOS (Intel and Apple Silicon), Linux, Both TCP and UDP are supported. -** When using Telepresence to ingest or intercept a container, are the Kubernetes cluster environment variables proxied on my local machine?** +** When using Telepresence run a cluster service locally, are the Kubernetes cluster environment variables proxied on my local machine?** -Yes, you can either set the container's environment variables on your machine or write the variables to a file to use with Docker or another build process. You can also directly pass the environments to an intercept handler that is run by the ingest or intercept. Please see [the environment variable reference doc](reference/environment.md) for more information. +Yes, you can either set the container's environment variables on your machine or write the variables to a file to use with Docker or another build process. You can also directly pass the environments to a handler that runs locally. Please see [the environment variable reference doc](reference/environment.md) for more information. -** When using Telepresence to ingest or intercept a container, can the associated container volume mounts also be mounted by my local machine?** +** When using Telepresence to run a cluster service locally, can the associated container volume mounts also be mounted by my local machine?** Yes, please see [the volume mounts reference doc](reference/volume.md) for more information. @@ -51,7 +55,7 @@ You can connect to cloud-based data stores and services that are directly addres -** Will Telepresence be able to ingest and intercept workloads running on a private cluster or cluster running within a virtual private cloud (VPC)?** +** Will Telepresence be able to engage with workloads running on a private cluster or cluster running within a virtual private cloud (VPC)?** Yes, but it doesn't need to have a publicly accessible IP address. @@ -65,11 +69,11 @@ The local daemon needs sudo to create a VIF (Virtual Network Interface) for outb A single `traffic-manager` service is deployed in the `ambassador` namespace within your cluster, and this manages resilient intercepts and connections between your local machine and the cluster. -A Traffic Agent container is injected per pod that is being intercepted. The first time an ingest or intercept is made on a workload, all pods associated with this workload will be restarted with the Traffic Agent automatically injected. +A Traffic Agent container is injected per pod that is being engaged. The first time a `replace`, an `ingest`, or an `intercept` is made on a workload, all pods associated with this workload will be restarted with the Traffic Agent automatically injected. ** How can I remove all the Telepresence components installed within my cluster?** -You can run the command `telepresence helm uninstall` to remove everything from the cluster, including the `traffic-manager`, and all the `traffic-agent` containers injected into each pod being intercepted. +You can run the command `telepresence helm uninstall` to remove everything from the cluster, including the `traffic-manager`, and all the `traffic-agent` containers injected into each pod being engaged. Also run `telepresence quit -s` to stop all local daemons running. diff --git a/docs/howtos/cluster-in-vm.md b/docs/howtos/cluster-in-vm.md index c73713a424..cd4ba13118 100644 --- a/docs/howtos/cluster-in-vm.md +++ b/docs/howtos/cluster-in-vm.md @@ -1,6 +1,6 @@ --- title: Host a cluster in Docker or a VM -description: Use Telepresence to intercept services in a cluster running in a hosted docker container or virtual +description: Use Telepresence to engage with services in a cluster running in a hosted docker container or virtual machine. hide_table_of_contents: true --- diff --git a/docs/concepts/docker.md b/docs/howtos/docker.md similarity index 52% rename from docs/concepts/docker.md rename to docs/howtos/docker.md index 7efef2c5b2..c40d2aaef3 100644 --- a/docs/concepts/docker.md +++ b/docs/howtos/docker.md @@ -2,12 +2,12 @@ title: "Using Telepresence with Docker" hide_table_of_contents: true --- -# Telepresence with Docker Golden Path +# Telepresence with Docker ## Why? -It can be tedious to adopt Telepresence across your organization, since in its handiest form, it requires admin access, and needs to get along with any exotic -networking setup that your company may have. +It can be tedious to adopt Telepresence across your organization, since in its handiest form, it requires admin access, +and needs to get along with any exotic networking setup that your company may have. If Docker is already approved in your organization, this Golden path should be considered. @@ -20,56 +20,61 @@ Thus removing the need for root access, making it easier to adopt as an organiza Let's illustrate with a quick demo, assuming a default Kubernetes context named default, and a simple HTTP service: -```cli +```console $ telepresence connect --docker -Connected to context default, namespace default (https://default.cluster.bakerstreet.io) +Connected to context default, namespace default (https://kubernetes.docker.internal:6443) +``` +This method limits the scope of the potential networking issues since everything stays inside Docker. The Telepresence daemon can be found under the name `tp--cn` when listing your containers. + +```console $ docker ps -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -7a0e01cab325 datawire/telepresence:2.12.1 "telepresence connec…" 18 seconds ago Up 16 seconds 127.0.0.1:58802->58802/tcp tp-default +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +540a3c12f45b ghcr.io/telepresenceio/telepresence:2.22.0 "telepresence connec…" 18 seconds ago Up 16 seconds 127.0.0.1:58802->58802/tcp tp-default-cn ``` -This method limits the scope of the potential networking issues since everything stays inside Docker. The Telepresence daemon can be found under the name `tp-` when listing your containers. - -Start an intercept and a corresponding intercept-handler: +Replace a container in the cluster and start a corresponding local container: ```cli -$ telepresence intercept echo-easy --port 8080:80 --docker-run -- jmalloc/echo-server -Using Deployment echo-easy - Intercept name : echo-easy - State : ACTIVE - Workload kind : Deployment - Destination : 127.0.0.1:8080 - Service Port Identifier: proxied - Intercepting : all TCP requests +$ telepresence replace echo-sc --docker-run -- ghcr.io/telepresenceio/echo-server:latest +Using Deployment echo-sc + Container name: echo-sc + State : ACTIVE + Workload kind : Deployment + Port forwards : 127.0.0.1 -> 127.0.0.1 + 8080 -> 8080 TCP Echo server listening on port 8080. ``` -Using `--docker-run` starts the local container that acts as the intercept handler so that it uses the same network as the container that runs the telepresence daemon. It will also -have the remote volumes mounted in the same way as the remote container that it intercepts. +Using `--docker-run` starts the local container that acts as the handler, so that it uses the same network as the +container that runs the telepresence daemon. It will also receive the same incoming traffic and have the remote volumes +mounted in the same way as the remote container that it replaces. If you want to curl your remote service, you'll need to do that from a container that shares the daemon container's network. Telepresence provides a `curl` command that will do just that. ```console -$ telepresence curl echo-easy +$ telepresence curl echo-sc % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed -100 99 100 99 0 0 21104 0 --:--:-- --:--:-- -Request served by 4b225bc8d6f1 +100 196 100 196 0 0 4232 0 --:--:-- --:--:-- --:--:-- 4260 +Request served by 540a3c12f45b -GET / HTTP/1.1 +Intercept id b0bd5e75-2618-4bef-ac4e-4c08c4b58ec7:echo-sc/echo-sc +Intercepted container "echo-sc" +HTTP/1.1 GET / -Host: echo-easy +Host: echo-sc +User-Agent: curl/8.11.1 Accept: */* -User-Agent: curl/8.6.0 --:--:-- 24750 ``` -Similarly, if you want to start your intercept handler manually using `docker run`, you must ensure that it shares the -daemon container's network. The most convenient way to do that is to use `telepresence docker-run`. +Similarly, if you want to start your container manually using `docker run`, you must ensure that it shares the +daemon container's network. The most convenient way to do that is to use the `--docker-run` flag as explained above, +but you can also start a container separately using `telepresence docker-run`. ```console -$ telepresence docker-run -e PORT=8080 jmalloc/echo-server +$ telepresence docker-run ghcr.io/telepresenceio/echo-server:latest Echo server listening on port 8080. ``` @@ -85,7 +90,7 @@ $ telepresence connect --docker --name beta --namespace beta Now, with two connections active, you must pass the flag `--use ` to other commands, e.g. ```console -$ telepresence intercept echo-easy --use alpha --port 8080:80 --docker-run -- jmalloc/echo-server +$ telepresence replace echo-easy --use alpha --docker-run -- ghcr.io/telepresenceio/echo-server:latest ``` ## Key learnings @@ -93,5 +98,4 @@ $ telepresence intercept echo-easy --use alpha --port 8080:80 --docker-run -- jm * Using the Docker mode of telepresence **does not require root access**, and makes it **easier** to adopt it across your organization. * It **limits the potential networking issues** you can encounter. * It **limits the potential mount issues** you can encounter. -* It **enables simultaneous intercepts in multiple namespaces**. -* It leverages **Docker** for your interceptor. +* It **enables simultaneous engagements in multiple namespaces**. diff --git a/docs/howtos/intercepts.md b/docs/howtos/intercepts.md index 6165255368..fca790e65a 100644 --- a/docs/howtos/intercepts.md +++ b/docs/howtos/intercepts.md @@ -6,9 +6,48 @@ hide_table_of_contents: true # Code and debug an application locally -Telepresence allows you to code and debug an application locally, while giving it access to resources in a remote cluster. You can either -do an _ingest_, to gain read-only access to a service, or an _intercept_, to provide read-write access to a service and also re-route -traffic intended for it to your workstation. +## Local Development Methods + +Telepresence offers three powerful ways to develop your services locally: + +### Replace +* **How it Works:** + - Replaces an existing container within your Kubernetes cluster with a Traffic Agent. + - Reroutes traffic intended for the replaced container to your local workstation. + - Makes the remote environment of the replaced container available to the local workstation. + - Provides read-write access to the volumes mounted by replaced container. +* **Impact:** + - A Traffic Agent is injected into the pods of the targeted workload. + - The replaced container is removed from the pods of the targeted workload. + - The replaced container is restored when the replace operation ends. +* **Use-cases:** + - You're working with message queue consumers and must stop the remote container. + - You're working with remote containers configured without incoming traffic. + +### Intercept +* **How it Works:** + - Intercepts requests destined for a specific service port (or ports) and reroutes them to the local workstation. + - Makes the remote environment of the targeted container available to the local workstation. + - Provides read-write access to the volumes mounted by the targeted container. +* **Impact:** + - A Traffic Agent is injected into the pods of the targeted workload. + - Intercepted traffic is rerouted to the local workstation and will no longer reach the remote service. + - All containers keep on running. +* **Use-cases:** + - Your main focus is the service API rather than the cluster's pods and containers. + - You want your local service to only receive specific ingress traffic, while other traffic must be untouched. + - You want your remote container to continue processing other requests or background tasks. + +### Ingest +* **How it Works:** + - Makes the remote environment of the ingested container available to the local workstation. + - Provides read-only access to the volumes mounted by replaced container. +* **Impact:** + - A Traffic Agent is injected into the pods of the targeted workload. + - No traffic is rerouted and all containers keep on running. +* **Use-cases:** + - You want to keep the impact of your local development to a minimum. + - You have don't need traffic being routed from the cluster, and read-only access to the container's volumes is ok. ## Prerequisites @@ -18,9 +57,7 @@ in several examples. OpenShift users can substitute oc [commands instead](https: This guide assumes you have an application represented by a Kubernetes deployment and service accessible publicly by an ingress controller, and that you can run a copy of that application on your laptop. -## Intercept your application - -### Running everything directly on the workstation +## Replace the cluster container This approach offers the benefit of direct cluster connectivity from your workstation, simplifying debugging and modification of your application within its familiar environment. However, it requires root access to configure @@ -43,38 +80,100 @@ network telepresence, and remote mounts must be made relative to a specific moun ```console $ telepresence list ... - example-app: ready to intercept (traffic-agent not yet installed) + deolpoyment example-app: ready to engage (traffic-agent not yet installed) ... ``` -3. Get the name of the port you want to intercept on your service: - `kubectl get service --output yaml`. +3. Get the name of the container you want to replace (output truncated for brewity) + ```console + $ kubectl describe deploy example-app + Name: example-app + Namespace: default + CreationTimestamp: Tue, 14 Jan 2025 03:49:29 +0100 + Labels: app=example-app + Annotations: deployment.kubernetes.io/revision: 1 + Selector: app=example-app + Replicas: 1 desired | 1 updated | 1 total | 0 available | 1 unavailable + StrategyType: RollingUpdate + MinReadySeconds: 0 + RollingUpdateStrategy: 25% max unavailable, 25% max surge + Pod Template: + Labels: app=example-app + Containers: + echo-server: + Image: ghcr.io/telepresencio/echo-server + Port: 8080/TCP + ``` + +4. Replace the container. Please note that the `--container echo-server` flag here is optional. It's only needed when the workload has more than one container: + ```console + $ telepresence replace example-app --container echo-server --env-file /tmp/example-app.env --mount /tmp/example-app-mounts + Using Deployment example-app + Container name : echo-server + State : ACTIVE + Workload kind : Deployment + Port forwards : 10.1.4.106 -> 127.0.0.1 + 8080 -> 8080 TCP + Volume Mount Point: /tmp/example-app-mounts + ``` + Your workstation is now ready. You can run the application using the environment in the `/tmp/example-app.env` file and the + mounts under `/tmp/example-app-mounts`. The application can listen to `localhost:8080` to receive traffic intended for the + replaced container. On the cluster side of things, a Traffic Agent container has replaced the `echo-server`. + + Telepresence assumes that you want all declared container ports to be mapped to their corresponding port on `localhost`. You + can change this with the `--port` flag. For example, `--port 1080:8080` will map the replaced containers port number `8080` + to `localhost:1080`. The `--port` can also be used when the container is known to listen to ports that are not declared in + the manifest. + +5. Query the cluster in which you replaced your application and verify your local instance being invoked. All the traffic previously routed to your Kubernetes Service is now routed to your local environment - If we assume that the service and deployment use the same name: +You can now: +- Make changes on the fly and see them reflected when interacting with your Kubernetes environment. +- Query services only exposed in your cluster's network. +- Set breakpoints in your IDE to investigate bugs. + +6. You end the replace operation with the command `telepresence leave example-app --container echo-server` + +## Ingest the cluster container +In some situations, you want to work and debug the code locally, and you want it to be able to access other services in the cluster, +but you don't wish to interfere with the targeted workload. This is where the `telepresence ingest` command comes into play. Just +like `replace` command, it will make the environment and mounted containers of the targeted container available locally, but it will +not replace the container nor will it intercept any of its traffic. + +This example assumes that you have the `example-app` deployment. + +1. Connect and run and start an ingest from `example-app`: ```console - $ kubectl get service example-app --output yaml - ... - ports: - - name: http - port: 80 - protocol: TCP - targetPort: http - ... + $ telepresence connect + Launching Telepresence User Daemon + Launching Telepresence Root Daemon + Connected to context xxx, namespace default (https://) + $ telepresence ingest example-app --container echo-server --env-file /tmp/example-app.env --mount /tmp/example-app-mounts + Using Deployment example-app + Container name : echo-server + Workload kind : Deployment + Volume Mount Point: /tmp/example-app-mounts ``` -4. Intercept all traffic going to the application in your cluster: - ``` - telepresence intercept --port [][:] --env-file `. - ``` +2. Start your local application using the environment variables retrieved and the volumes that were mounted in the previous step. + +You can now: +- Code and debug your local app while it interacts with other services in your cluster. +- Query services only exposed in your cluster's network. +- Set breakpoints in your IDE to investigate bugs. - * For `--port`: specify the port the local instance of your application is running on. If the intercepted service exposes multiple ports, specify the port you want to intercept after a colon. - * For `--env-file`: specify a file path for Telepresence to write the environment variables that are set for the intercepted container. +## Intercept your application + +You can use the `telepresence intercept` command when you want to intercept the traffic for a specific service and route that +traffic to your workstation. The `intercept` is less intrusive than the `replace`, because it allows the original receiver of the +intercepted traffic to continue to run and deal with tasks that aren't directly related to that traffic. - The example below shows Telepresence intercepting traffic going to deployment `example-app`. Requests to the service on port `http` in the cluster get routed to `8080` on the workstation and the environment variables of the service are written to `~/example-app-intercept.env`. +1. Connect to your cluster with `telepresence connect`. +2. Intercept all traffic going to the application's http port in your cluster and redirect to port 8080 on your workstation. ```console - $ telepresence intercept example-app --port 8080:http --env-file ~/example-app-intercept.env + $ telepresence intercept example-app --port 8080:http --env-file ~/example-app-intercept.env --mount /tmp/example-app-mounts Using Deployment example-app intercepted Intercept name: example-app @@ -84,29 +183,34 @@ network telepresence, and remote mounts must be made relative to a specific moun Intercepting : all TCP connections ``` -5. Start your local application using the environment variables retrieved in the previous step. - The following are some examples of how to pass the environment variables to your local process: - * **Visual Studio Code:** specify the path to the environment variables file in the `envFile` field of your configuration. - * **JetBrains IDE (IntelliJ, WebStorm, PyCharm, GoLand, etc.):** use the [EnvFile plugin](https://plugins.jetbrains.com/plugin/7861-envfile). + * For `--port`: specify the port the local instance of your application is running on, and optionally the remote port + that you want to intercept. Telepresence will select the remote port automatically when there's only one service + port available to access the workload. You must specify the port to intercept when the workload exposes multiple + ports. You can do this by specifying the port you want to intercept after a colon in the `--port` argument (like in + the example), and/or by specifying the service you want to intercept using the `--service` flag. + + * For `--env-file`: specify a file path for Telepresence to write the environment variables that are set for the targeted + container. -6. Query the cluster in which you intercepted an application and verify your local instance being invoked. - All the traffic previously routed to your Kubernetes Service is now routed to your local environment +3. Start your local application using the environment variables retrieved and the volumes that were mounted in the previous step. You can now: -- Make changes on the fly and see them reflected when interacting with - your Kubernetes environment. +- Make changes on the fly and see them reflected when interacting with your Kubernetes environment. - Query services only exposed in your cluster's network. - Set breakpoints in your IDE to investigate bugs. ### Running everything using Docker -This approach eliminates the need for root access and confines the Telepresence network interface to a container. -Additionally, it allows for precise replication of the target container's volume mounts, using identical mount points. -However, this method sacrifices direct cluster connectivity from the workstation and the containerized environment can +This approach eliminates the need for root access and confines the Telepresence network interface and remote mounts +to a container. Additionally, it allows for precise replication of the target container's volume mounts, using identical +mount points. However, this method will require docker to get cluster connectivity, and the containerized environment can present challenges in terms of toolchain integration, debugging, and the overall development workflow. - -1. Connect to your cluster with `telepresence connect --docker` and try to curl the Kubernetes API server from a container. A 401 or 403 response code is expected and indicates that the service could be reached. The `telepresence curl` command used here is the same as `curl` but uses a container initiated with the network created by the Telepresence daemon: +1. Connect to your cluster with `telepresence connect --docker`. This starts the Telepresence daemon in a docker + container and ensures that this container has access to the cluster network. +2. Use `telepresence curl` to access the Kubernetes API server from a container. + A 401 or 403 response code is expected and indicates that the service could be reached. The `telepresence curl` command + used will execute a standard `curl` command from a container that shares the network created by the `connect` call: ```console $ telepresence curl -ik https://kubernetes.default @@ -116,47 +220,22 @@ present challenges in terms of toolchain integration, debugging, and the overall ... ``` - You now have access to your remote Kubernetes API server as if you were on the same network. You can now use any local tools to connect to any service in the cluster. + You now have access to your remote Kubernetes API server as if you were on the same network. -2. Enter `telepresence list` and make sure the workload (deployment in this case) you want to intercept is listed. For example: +3. Enter `telepresence list` and make sure the workload you want to engage is listed. For example: ```console $ telepresence list ... - example-app: ready to intercept (traffic-agent not yet installed) + deployment example-app: ready to engage (traffic-agent not yet installed) ... ``` -3. Get the name of the port you want to intercept on your service: - `kubectl get service --output yaml`. - - If we assume that the service and deployment use the same name: - - ```console - $ kubectl get service example-app --output yaml - ... - ports: - - name: http - port: 80 - protocol: TCP - targetPort: http - ... - ``` - -4. Intercept all traffic going to the application in your cluster, and start a local container to handle that intercept: - ``` - telepresence intercept --port [][:] --docker-run -- . - ``` - - * For `--port`: If the intercepted service exposes multiple ports, specify the service port you want to intercept after a colon. - The local port can be empty to default to the same as the targeted container port. - - The example below shows Telepresence intercepting traffic going to deployment `example-app`. The local container inherits - the environment and the volume mounts from the targeted container, and requests to the service on port `http` in the - cluster get routed to the local container and the environment variables of the service are written to `~/example-app-intercept.env`. +4. Use `replace`, `inject`, or `intercept` to engage the container in combination with the `--docker-run` flag. + Example using `telepresence replace` ```console - $ telepresence intercept example-app --port :http --docker-run -- + $ telepresence replace example-app --container echo-server --docker-run -- Using Deployment example-app intercepted Intercept name: example-app @@ -167,59 +246,8 @@ present challenges in terms of toolchain integration, debugging, and the overall ``` -5. Query the cluster in which you intercepted an application and verify your local instance being invoked. - All the traffic previously routed to your Kubernetes Service is now routed to your local container. - You can now: - Make changes on the fly and see them reflected when interacting with your Kubernetes environment; although depending on how your local container is configured, this might require that it is rebuilt. - Query services only exposed in your cluster's network using `telepresence curl`. - Set breakpoints in a _Remote Debug_ configuration in your IDE to investigate bugs. - -## Ingest your service - -In some situations, you want to work and debug the code locally, and you want it to be able to access other services in the cluster, -but you don't wish to intercept any traffic intended for the targeted workload. This is where the `telepresence ingest` command -comes into play. Just like intercept, it will make the environment and mounted containers of the targeted container available locally, -but it will not intercept any traffic. - -This example assumes that you have the `example-app` - -### Running everything directly on the workstation - -1. Connect and run and start an ingest from `example-app`: - ```console - $ telepresence connect - Launching Telepresence User Daemon - Launching Telepresence Root Daemon - Connected to context xxx, namespace default (https://) - $ telepresence ingest example-app --env-file ~/example-app-intercept.env - Using Deployment example-app - Container : example-app - Volume Mount Point: /tmp/telfs-166994305 - ``` - -2. Start your local application using the environment variables retrieved in the previous step. - -You can now: -- Code and debug your local app while it interacts with other services in your cluster. -- Query services only exposed in your cluster's network. -- Set breakpoints in your IDE to investigate bugs. - -### Running everything using Docker - -1. Connect using docker start an ingest from `example-app`, and run a container locally with the ingested environment and volume mounts: - ```console - $ telepresence connect --docker - Launching Telepresence User Daemon - Connected to context xxx, namespace default (https://) - $ telepresence ingest example-app --expose 8080 --docker-run -- - Using Deployment example-app, container example-app - - ``` - -You can now: -- Code and debug your local container while it interacts with other services in your cluster. -- Send request to your local container using localhost:<local port> -- Query services only exposed in your cluster's network using `telepresence curl`. -- Set breakpoints in a _Remote Debug_ configuration in your IDE to investigate bugs. diff --git a/docs/howtos/large-clusters.md b/docs/howtos/large-clusters.md index 42e4554b26..ac755fbc23 100644 --- a/docs/howtos/large-clusters.md +++ b/docs/howtos/large-clusters.md @@ -20,7 +20,7 @@ The `telepresence connect` command will accept the flag `--mapped-namespaces - - ``` - - ```console - $ curl -ik https://kubernetes.default - HTTP/1.1 401 Unauthorized - Cache-Control: no-cache, private - Content-Type: application/json - ... - - ``` - - You now have access to your remote Kubernetes API server as if you were on the same network. You can now use any local tools to connect to any service in the cluster. - -2. Enter `telepresence list` and make sure the service you want to intercept is listed. For example: - - ```console - $ telepresence list - ... - example-service: ready to intercept (traffic-agent not yet installed) - ... - ``` - -3. Get the name of the port you want to intercept on your service: - `kubectl get service --output yaml`. - - For example: - - ```console - $ kubectl get service example-service --output yaml - ... - ports: - - name: http - port: 80 - protocol: TCP - targetPort: http - ... - ``` - -4. Intercept all traffic going to the service in your cluster: - - ```console - $ telepresence intercept --port [:] --env-file ` - ``` - - - For `--port`: specify the port the local instance of your service is running on. If the intercepted service exposes multiple ports, specify the port you want to intercept after a colon. - - For `--env-file`: specify a file path for Telepresence to write the environment variables that are set in the pod. - The example below shows Telepresence intercepting traffic going to service `example-service`. Requests now reach the service on port `http` in the cluster get routed to `8080` on the workstation and write the environment variables of the service to `~/example-service-intercept.env`. - - ``` - $ telepresence intercept example-service --port 8080:http --env-file ~/example-service-intercept.env - Using Deployment example-service - intercepted - Intercept name: example-service - State : ACTIVE - Workload kind : Deployment - Destination : 127.0.0.1:8080 - Intercepting : all TCP connections - ``` - -5. Start your local environment using the environment variables retrieved in the previous step. - -The following are some examples of how to pass the environment variables to your local process: - -- **Docker:** enter `docker run` and provide the path to the file using the `--env-file` argument. For more information about Docker run commands, see the [Docker command-line reference documentation](https://docs.docker.com/engine/reference/commandline/run/#env). -- **Visual Studio Code:** specify the path to the environment variables file in the `envFile` field of your configuration. -- **JetBrains IDE (IntelliJ, WebStorm, PyCharm, GoLand, etc.):** use the [EnvFile plugin](https://plugins.jetbrains.com/plugin/7861-envfile). - -6. Query the environment in which you intercepted a service and verify your local instance being invoked. - All the traffic previously routed to your Kubernetes Service is now routed to your local environment - -## 🎉 You've Unlocked a Faster Development Workflow for Kubernetes with Telepresence - -Now, with Telepresence, you can: - -- Make changes on the fly and see them reflected when interacting with your remote Kubernetes environment, this is just like hot reloading, but it works across both local and remote environments. -- Query services and microservice APIs that are only accessible in your remote cluster's network. -- Set breakpoints in your IDE and re-route remote traffic to your local machine to investigate bugs with realistic user traffic and API calls. - -> [!TIP] -> **Didn't work?** Make sure the port you're listening on matches the one you specified when you created your intercept. +Checkout the [Howto](howtos/intercepts.md) to learn how Telepresence can engage with resources in your remote cluster, +enabling you to run the code on your local workstation. ## What’s Next? - [Learn about the Telepresence architecture.](reference/architecture) diff --git a/docs/reference/architecture.md b/docs/reference/architecture.md index 93e298538d..a19fa228a4 100644 --- a/docs/reference/architecture.md +++ b/docs/reference/architecture.md @@ -18,7 +18,7 @@ Telepresence has Daemons that run on a developer's workstation and act as the ma network in order to communicate with the cluster and handle intercepted traffic. ### User-Daemon -The User-Daemon coordinates the creation and deletion of ingests and intercepts by communicating with the [Traffic Manager](#traffic-manager). +The User-Daemon coordinates the creation and deletion of replacements, ingests and intercepts by communicating with the [Traffic Manager](#traffic-manager). All requests from and to the cluster go through this Daemon. ### Root-Daemon @@ -30,19 +30,19 @@ please refer to this blog post: ## Traffic Manager The Traffic Manager is the central point of communication between Traffic Agents in the cluster and Telepresence Daemons -on developer workstations. It is responsible for injecting the Traffic Agent sidecar into ingested or intercepted pods, -proxying all relevant inbound and outbound traffic, and tracking active intercepts. +on developer workstations. It is responsible for injecting the Traffic Agent sidecar into engaged pods, +proxying all relevant inbound and outbound traffic, and tracking active engagements. The Traffic-Manager is installed by a cluster administrator. It can either be installed using the Helm chart embedded in the telepresence client binary (`telepresence helm install`) or by using a Helm Chart directly. ## Traffic Agent -The Traffic Agent is a sidecar container that facilitates ingests and intercepts. When an ingest or intercept is first +The Traffic Agent is a sidecar container that facilitates engagements. When a `replace`, `ingest` or `intercept` is first started, the Traffic Agent container is injected into the workload's pod(s). You can see the Traffic Agent's status by running `telepresence list` or `kubectl describe pod `. -Depending on if an intercept is active or not, the Traffic Agent will either route the incoming request to a -your workstation, or it will pass it along to the container in the pod usually handling requests on that port. +Depending on if an `replace` or `intercept` is active or not, the Traffic Agent will either route the incoming request +to your workstation, or it will pass it along to the container in the pod usually handling requests. Please see [Traffic Agent Sidecar](intercepts/sidecar.md) for details. \ No newline at end of file diff --git a/docs/reference/client.md b/docs/reference/client.md index 73aa5e13d0..b6ff818c27 100644 --- a/docs/reference/client.md +++ b/docs/reference/client.md @@ -1,6 +1,6 @@ --- title: Client reference -description: CLI options for Telepresence to intercept traffic from your Kubernetes cluster to code running on your laptop. +description: CLI options for Telepresence to engage with resources in your Kubernetes cluster with code running on your laptop. hide_table_of_contents: true --- @@ -27,10 +27,12 @@ You can append `--help` to each command below to get even more information about | `ingest` | Ingest a container to get access to its mounted volumes and environment variables: `telepresence ingest --container --env-file ` When used with a `--` separator, this command can also start a process so you can run a local instance of the ingested container. | | | `intercept` | Intercepts a service to get its ingress traffic routed to the workstation and access to its mounted volumes and environment variables: `telepresence intercept --port ` (use `port/UDP` to force UDP). When used with a `--` separator, this command can also start a process so you can run a local instance of the service you are intercepting. | -| `leave` | Stops an active ingest or intercept: `telepresence leave hello`. | -| `list` | Lists all workloads that are eligible for ingest or intercept. | +| `leave` | Stops an active replace, ingest, or intercept: `telepresence leave hello`. | +| `list` | Lists all workloads that are eligible for replace, ingest, or intercept. | | `loglevel` | Temporarily change the log-level. The default duration (30 minutes) can be altered using `-d `. The flags `--local-only` and `--remote-only` can be used to alter the scope of the change. | | `quit` | Tell Telepresence daemons to quit. | +| `replace` | Replace a container to get access to its traffic, mounted volumes and environment variables: `telepresence replace --container --env-file ` When used with a `--` separator, this command can also start a process so you can run a local instance of the replaced container. | +| | `status` | Shows the current connectivity status. | | `uninstall` | Uninstalls a Traffic Agent for a specific workload. Use the `--all-agents` flag to remove all Traffic Agents from all workloads. | | `version` | Show version of Telepresence CLI + Traffic-Manager (if connected) | \ No newline at end of file diff --git a/docs/reference/cluster-config.md b/docs/reference/cluster-config.md index 16a813109a..a21116d79c 100644 --- a/docs/reference/cluster-config.md +++ b/docs/reference/cluster-config.md @@ -53,10 +53,10 @@ The `agent.resources` and `agent.initResources` will be used as the `resources` ## Mutating Webhook Telepresence uses a Mutating Webhook to inject the [Traffic Agent](architecture.md#traffic-agent) sidecar container and update the -port definitions. This means that an ingested or intercepted workload (Deployment, StatefulSet, ReplicaSet, ArgoRollout) will remain untouched +port definitions. This means that an engaged workload (Deployment, StatefulSet, ReplicaSet, ArgoRollout) will remain untouched and in sync as far as GitOps workflows (such as ArgoCD) are concerned. -The injection will happen on demand the first time an attempt is made to ingest or intercept the workload. +The injection will happen on demand the first time an attempt is made to replace, ingest, or intercept the workload. If you want to prevent that the injection ever happens, simply add the `telepresence.getambassador.io/inject-traffic-agent: disabled` annotation to your workload template's annotations: diff --git a/docs/reference/config.md b/docs/reference/config.md index c4bf07a0ea..1cb21cfcee 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -28,7 +28,7 @@ client: userDaemon: debug images: registry: privateRepo # This overrides the default docker.io/datawire repo - agentImage: tel2:$version$ # This overrides the agent image to inject when intercepting + agentImage: tel2:$version$ # This overrides the agent image to inject when engaging with a workload grpc: maxReceiveSize: 10Mi dns: @@ -124,7 +124,7 @@ These are the valid fields for the `client.images` key: ### Intercept -The `intercept` controls applies to how Telepresence will intercept the communications to the intercepted service. +The `intercept` controls applies to how Telepresence will intercept the communications to replaced containers and intercepted services. | Field | Description | Type | Default | |-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|--------------| @@ -270,7 +270,7 @@ logLevels: userDaemon: debug images: registry: privateRepo # This overrides the default docker.io/datawire repo - agentImage: tel2:$version$ # This overrides the agent image to inject when intercepting + agentImage: tel2:$version$ # This overrides the agent image to inject when engaging with a workload grpc: maxReceiveSize: 10Mi ``` diff --git a/docs/reference/dns.md b/docs/reference/dns.md index 3265a44457..64469844b1 100644 --- a/docs/reference/dns.md +++ b/docs/reference/dns.md @@ -4,27 +4,23 @@ hide_table_of_contents: true --- # DNS resolution -The Telepresence DNS resolver is dynamically configured to resolve names using the namespaces of currently active intercepts. Processes running locally on the desktop will have network access to all services in the such namespaces by service-name only. - -All intercepts contribute to the DNS resolver, even those that do not use the `--namespace=` option. This is because `--namespace default` is implied, and in this context, `default` is treated just like any other namespace. - -No namespaces are used by the DNS resolver (not even `default`) when no intercepts are active, which means that no service is available by `` only. Without an active intercept, the namespace qualified DNS name must be used (in the form `.`). +The Telepresence DNS resolver is dynamically configured to resolve names using the namespaces currently managed by the Traffic Manager. Processes running locally on the desktop will have network access to all services in the currently connected namespace by service-name only, and to other managed namespaces using service-name.namespace. See this demonstrated below, using the [quick start's](../quick-start.md) sample app services. -No intercepts are currently running, we'll connect to the cluster and list the services that can be intercepted. +We'll connect to a namespace in the cluster and list the services that can be intercepted. ``` -$ telepresence connect +$ telepresence connect --namespace default Connecting to traffic manager... Connected to context default, namespace default (https://) $ telepresence list - web-app: ready to intercept (traffic-agent not yet installed) - emoji : ready to intercept (traffic-agent not yet installed) - web : ready to intercept (traffic-agent not yet installed) + deployment web-app: ready to engage (traffic-agent not yet installed) + deployment emoji : ready to engage (traffic-agent not yet installed) + deployment web : ready to engage (traffic-agent not yet installed) $ curl web-app:80 @@ -36,32 +32,8 @@ $ curl web-app:80 ... ``` -Now we'll start an intercept against another service. - -``` -$ telepresence intercept web --port 8080 - - Using Deployment web - intercepted - Intercept name : web - State : ACTIVE - Workload kind : Deployment - Destination : 127.0.0.1:8080 - Volume Mount Point: /tmp/telfs-166119801 - Intercepting : all TCP connections - -$ curl webapp:80 - - - - - - Emoji Vote - ... -``` - The DNS resolver will also be able to resolve services using `.` regardless of what namespace the -client is connected to. +client is connected to as long as the given namespace is among the set managed by the Traffic Manager. ### Supported Query Types diff --git a/docs/reference/docker-run.md b/docs/reference/docker-run.md index 1a6003fce5..e70595a9ca 100644 --- a/docs/reference/docker-run.md +++ b/docs/reference/docker-run.md @@ -5,7 +5,7 @@ toc_min_heading_level: 2 toc_max_heading_level: 2 --- -# Using Docker for intercepts +# Using Docker when engaging with workloads ## Using command flags @@ -16,8 +16,6 @@ You can start the Telepresence daemon in a Docker container on your laptop using $ telepresence connect --docker ``` -The `--docker` flag is a global flag, and if passed directly like `telepresence intercept --docker`, then the implicit connect that takes place if no connections are active, will use a container-based daemon. - ### The telepresence curl command The network interface that is added when connecting using `telepresence connect --docker` will not be accessible directly from the host computer. It is confined to the telepresence daemon container, and there you should not expect to be able to curl your cluster resources directly. @@ -39,9 +37,10 @@ ensuring that the desired ports are exposed to the local environment. > was in effect when it started, then it will lose its network. In other words, when using `telepresence docker-run`, > you must always rerun after a `telepresence quit`/`telepresence connect --docker`. -### The ingest/intercept --docker-run flag +### The replace/ingest/intercept --docker-run flag -If you want your ingest or intercept to use another Docker container, you can use the `--docker-run` flag. It creates the ingest or intercept, runs your container in the foreground, then automatically ends the ingest or intercept when the container exits. +If you want your replace, ingest, or intercept to use another Docker container, you can use the `--docker-run` flag. It will establish the engagement, +run your container in the foreground, then automatically end the engagement when the container exits. After establishing a connection to a cluster using `telepresence connect --docker`, the container started when using `--docker-run` will share the same network as the containerized daemon that maintains the connection. This enables seamless communication between your local development @@ -50,14 +49,18 @@ environment and the remote cluster. The `docker run` flags `--network`, `--publish`, or `--expose` are all available, just as with the `docker-run` command. ```console -$ telepresence intercept --port --docker-run -- +$ telepresence replace --container --docker-run -- ``` OR ```console $ telepresence ingest --container --docker-run -- ``` +OR +```console +$ telepresence intercept --port --docker-run -- +``` -The `--` separates flags intended for `telepresence ingest/intercept` from flags intended for `docker run`. +The `--` separates flags intended for `telepresence replace/ingest/intercept` from flags intended for `docker run`. It's recommended that you always use the `--docker-run` in combination with a connection started with the `telepresence connect --docker`, because that makes everything less intrusive: @@ -67,15 +70,15 @@ because that makes everything less intrusive: The following happens under the hood when both flags are in use: -- The network of for the ingest or intercept handler will be set to the same as the network used by the daemon. This guarantees that the - ingest or intercept handler can access the Telepresence VIF, and hence have access the cluster. +- The network of for the replace, ingest, or intercept handler will be set to the same as the network used by the daemon. This guarantees that the + handler can access the Telepresence VIF, and hence have access the cluster. - Volume mounts will be automatic and made using the Telemount Docker volume plugin so that all volumes exposed by the targeted remote container are mounted on the local handler container. - The environment of the remote container becomes the environment of the local handler container. ### The docker-build flag -The `--docker-build ` and the repeatable `docker-build-opt key=value` flags enable container's to be build on the fly by the intercept command. +The `--docker-build ` and the repeatable `docker-build-opt key=value` flags enable container's to be build on the fly by the replace/ingest/intercept command. When using `--docker-build`, the image name used in the argument list must be verbatim `IMAGE`. The word acts as a placeholder and will be replaced by the ID of the image that is built. @@ -85,7 +88,7 @@ The `--docker-build` flag implies `--docker-run`. It is possible to use `--docker-run` with a daemon running on your host, which is the default behavior of Telepresence. -However, it isn't recommended since you'll be in a hybrid mode: while your intercept runs in a container, the daemon will modify the host network, and if remote mounts are desired, they may require extra software. +However, it isn't recommended since you'll be in a hybrid mode: while your handler runs in a container, the daemon will modify the host network, and if remote mounts are desired, they may require extra software. The ability to use this special combination is retained for backward compatibility reasons. It might be removed in a future release of Telepresence. @@ -97,28 +100,29 @@ is done using `--port :`. The container port will de Imagine you are working on a new version of your frontend service. It is running in your cluster as a Deployment called `frontend-v1`. You use Docker on your laptop to build an improved version of the container called `frontend-v2`. To test it out, use this command to run the new container on your laptop and start an intercept of the cluster service to your local container. ```console -$ telepresence intercept --docker frontend-v1 --port 8000 --docker-run -- frontend-v2 +$ telepresence connect --docker +$ telepresence replace frontend-v1 --docker-run -- frontend-v2 ``` -Now, imagine that the `frontend-v2` image is built by a `Dockerfile` that resides in the directory `images/frontend-v2`. You can build and intercept directly. +Now, imagine that the `frontend-v2` image is built by a `Dockerfile` that resides in the directory `images/frontend-v2`. You can build and replace directly. ```console -$ telepresence intercept --docker frontend-v1 --port8000 --docker-build images/frontend-v2 --docker-build-opt tag=mytag -- IMAGE +$ telepresence replace frontend-v1 --docker-build images/frontend-v2 --docker-build-opt tag=mytag -- IMAGE ``` ## Automatic flags -Telepresence will automatically pass some relevant flags to Docker in order to connect the container with the intercept. Those flags are combined with the arguments given after `--` on the command line. +Telepresence will automatically pass some relevant flags to Docker to connect the container with the remote container. Those flags are combined with the arguments given after `--` on the command line. -- `--env-file ` Loads the intercepted environment +- `--env-file ` Loads the remote environment - `--name intercept--` Names the Docker container, this flag is omitted if explicitly given on the command line - `-v ` Volume mount specification, see CLI help for `--docker-mount` flags for more info When used with a container based daemon: - `--rm` Mandatory, because the volume mounts cannot be removed until the container is removed. -- `-v :` Volume mount specifications propagated from the intercepted container +- `-v :` Volume mount specifications propagated from the engaged container - `--network container:` Network is shared with the containerized daemon When used with a daemon that isn't container based: -- `--dns-search tel2-search` Enables single label name lookups in intercepted namespaces +- `--dns-search tel2-search` Enables single label name lookups in the connected namespace - `-p ` The local port for the intercept and the container port diff --git a/docs/reference/environment.md b/docs/reference/environment.md index 7fb6796827..f6418439d7 100644 --- a/docs/reference/environment.md +++ b/docs/reference/environment.md @@ -6,38 +6,38 @@ hide_table_of_contents: true # Environment variables -Telepresence will import environment variables from the cluster container when running an ingest or intercept. +Telepresence will import environment variables from the cluster container when engaging with it. You can use these variables with the code running on your laptop. There are several options available to do this: -1. `telepresence intercept [service] --port [port] --env-file=[FILENAME]` +1. `telepresence replace --container --env-file ` This will write the environment variables to a file. This file can be used when starting containers locally. The option `--env-syntax` will allow control over the syntax of the file. Valid syntaxes are "docker", "compose", "sh", "csh", "cmd", and "ps" where "sh", "csh", and "ps" can be suffixed with ":export". -2. `telepresence intercept [service] --port [port] --env-file=[FILENAME] --env-syntax=json` +2. `telepresence replace --container --env-file --env-syntax=json` This will write the environment variables to a JSON file. This file can be injected into other build processes. -3. `telepresence intercept [service] --port [port] -- [COMMAND]` +3. `telepresence replace --container -- ` - This will run a command locally with the pod's environment variables set on your laptop. Once the command quits the intercept is stopped (as if `telepresence leave [service]` was run). This can be used in conjunction with a local server command, such as `python [FILENAME]` or `node [FILENAME]` to run a service locally while using the environment variables that were set on the pod via a ConfigMap or other means. + This will run a command locally with the pod's environment variables set on your laptop. Once the command quits the `replace` is stopped (as if `telepresence leave ` was run). This can be used in conjunction with a local server command, such as `python [FILENAME]` or `node [FILENAME]` to run a service locally while using the environment variables that were set on the pod via a ConfigMap or other means. Another use would be running a subshell, Bash for example: -4. `telepresence intercept [service] --port [port] -- /bin/bash` +4. `telepresence replace -- /bin/bash` - This would start the intercept then launch the subshell on your laptop with all the same variables set as on the pod. + This would start the `replace` and then launch the subshell on your laptop with all the same variables set as on the pod. -5. `telepresence intercept [service] --docker-run -- [CONTAINER]` +5. `telepresence replace --docker-run -- ` This will ensure that the environment is propagated to the container. Will also work for `--docker-build` and `--docker-debug`. ## Telepresence Environment Variables -Telepresence adds some useful environment variables in addition to the ones imported from the intercepted pod: +Telepresence adds some useful environment variables in addition to the ones imported from the engaged container: ### TELEPRESENCE_ROOT Directory where all remote volumes mounts are rooted. See [Volume Mounts](volume.md) for more info. @@ -46,4 +46,4 @@ Directory where all remote volumes mounts are rooted. See [Volume Mounts](volume Colon separated list of remotely mounted directories. ### TELEPRESENCE_CONTAINER -The name of the intercepted container. Useful when a pod has several containers, and you want to know which one that was intercepted by Telepresence. +The name of the targeted container. Useful when a pod has several containers, and you want to know which one that was engaged by Telepresence. diff --git a/docs/reference/inside-container.md b/docs/reference/inside-container.md index db15d0af1c..8059b9403d 100644 --- a/docs/reference/inside-container.md +++ b/docs/reference/inside-container.md @@ -4,7 +4,7 @@ hide_table_of_contents: true --- # Running Telepresence inside a container -## Run with the daemon and intercept handler in containers +## Run with the daemon and engagement handler in containers The `telepresence connect` command now has the option `--docker`. This option tells telepresence to start the Telepresence daemon in a docker container. @@ -13,8 +13,8 @@ Running the daemon in a container brings many advantages. The daemon will no lon it will not mount files in the host's filesystem. Consequently, it will not need admin privileges to run, nor will it need special software like macFUSE or WinFSP to mount the remote file systems. -The intercept handler (the process that will receive the intercepted traffic) must also be a docker container, because that is the only -way to access the cluster network that the daemon makes available, and to mount the docker volumes needed. +The engagement handler (the process that runs locally and optionally will receive intercepted traffic) must also be a docker container, +because that is the only way to access the cluster network that the daemon makes available, and to mount the docker volumes needed. ## Run everything in a container diff --git a/docs/reference/intercepts/cli.md b/docs/reference/intercepts/cli.md index 8de626f79a..07888958a0 100644 --- a/docs/reference/intercepts/cli.md +++ b/docs/reference/intercepts/cli.md @@ -1,27 +1,27 @@ --- -title: Configure intercept using CLI +title: Configure workload engagements using CLI --- -# Configuring intercept using CLI +# Configuring workload engagements using CLI -## Specifying a namespace for an intercept +## Specifying a namespace for an engagement -The namespace of the intercepted workload is specified during connect using the `--namespace` option. +The namespace of the engaged workload is specified during connect using the `--namespace` option. ```shell telepresence connect --namespace myns -telepresence intercept hello --port 9000 +telepresence replace/ingest/intercept hello ``` ## Importing environment variables Telepresence can import the environment variables from the pod that is -being intercepted, see [this doc](../environment.md) for more details. +being engaged, see [this doc](../environment.md) for more details. ## Creating an intercept The following command will intercept all traffic bound to the service and proxy it to your -laptop. This includes traffic coming through your ingress controller, so use this option +laptop. This includes traffic coming through your ingress controller, so use this option carefully as to not disrupt production environments. ```shell @@ -292,7 +292,7 @@ port, optionally suffixed with `/TCP` or `/UDP` ```console $ telepresence list - echo-no-svc: ready to intercept (traffic-agent not yet installed) + deployment echo-no-svc: ready to engage (traffic-agent not yet installed) ``` 4. Start an intercept handler locally that will receive the incoming traffic. Here's an example using a simple python http service: diff --git a/docs/reference/intercepts/container.md b/docs/reference/intercepts/container.md index 1d99698e0e..ee213eedcf 100644 --- a/docs/reference/intercepts/container.md +++ b/docs/reference/intercepts/container.md @@ -3,12 +3,15 @@ title: Target a specific container --- # Target a specific container -An intercept ultimately targets a specific port within a container. The port is usually determined +A `telepresence replace` or `telepresence ingest` will always target a specific container, and the `--container` flag is +mandatory when the workload has more than one container. + +A `telepresence intercept` will ultimately target a specific port within a container. The port is usually determined by examining the relationship between the service's `targetPort` and the container's `containerPort`. In certain scenarios, the container owning the intercepted port differs from the container the intercept targets. This container's sole purpose is to route traffic from the service to the intended container, -often using a direct localhost connection. +often using a direct localhost connection. Use the `--container` flag with the intercept in these scenarios. ## No intercept @@ -18,6 +21,10 @@ Consider the following scenario: ## Standard Intercept +During a replace, the Telepresence traffic-agent will redirect all traffic intended for the replaced container to the +workstation. It will also make the environment and mounts for the **Nginx container** available, because it is +considered to be the one targeted by the intercept. + During an intercept, the Telepresence traffic-agent will redirect the `http` port to the workstation. It will also make the environment and mounts for the **Nginx container** available, because it is considered to be the one targeted by the intercept. diff --git a/docs/reference/intercepts/sidecar.md b/docs/reference/intercepts/sidecar.md index a3a474eb2c..493b15911c 100644 --- a/docs/reference/intercepts/sidecar.md +++ b/docs/reference/intercepts/sidecar.md @@ -3,16 +3,18 @@ title: Traffic Agent Sidecar --- # Traffic Agent Sidecar -When intercepting a service, the Telepresence Traffic Manager ensures -that a Traffic Agent has been injected into the intercepted workload. +When replacing a container or intercepting a service, the Telepresence Traffic Manager ensures +that a Traffic Agent has been injected into the targeted workload. The injection is triggered by a Kubernetes Mutating Webhook and will -only happen once. The Traffic Agent is responsible for redirecting -intercepted traffic to the developer's workstation. +only happen once. The Traffic Agent is responsible for making the environment and volumes available +on the developer's workstation, and also for redirecting traffic to it. -The intercept will intercept all `tcp` and/or `udp` traffic to the -intercepted service and send all of that traffic down to the developer's -workstation. This means that an intercept will affect all users of -the intercepted service. +When replacing a workload container, all traffic intended for it will be rerouted to the local workstation, unless +limited using the `--port` flag. + +When intercepting, all `tcp` and/or `udp` traffic to the targeted port is sent to the developer's workstation. + +This means that both a `replace` and an `intercept` will affect all users of the targeted workload. ## Supported workloads @@ -20,8 +22,8 @@ Kubernetes has various [workloads](https://kubernetes.io/docs/concepts/workloads/). Currently, Telepresence supports installing a Traffic Agent container on `Deployments`, `ReplicaSets`, `StatefulSets`, and `ArgoRollouts`. A Traffic Agent is -installed the first time a user makes a `telepresence ingest WORKLOAD`, `telepresence intercept WORKLOAD`, or a -`telepresence connect --proxy-via CIDR=WORKLAOD`. +installed the first time a user makes a `telepresence replace WORKLOAD`, `telepresence ingest WORKLOAD`, +`telepresence intercept WORKLOAD`, or a `telepresence connect --proxy-via CIDR=WORKLAOD`. A Traffic Agent may also be installed up front by adding a `telepresence.getambassador.io/inject-traffic-agent: enabled` annotation to the WORKLOADS pod template. @@ -54,7 +56,7 @@ require a Traffic Agent. ### Disable workloads By default, traffic-manager will observe `Deployments`, `ReplicaSets` and `StatefulSets`. -Each workload used today adds certain overhead. If you are not intercepting a specific workload type, you can disable it to reduce that overhead. +Each workload used today adds certain overhead. If you are not engaging a specific workload type, you can disable it to reduce that overhead. That can be achieved by setting the Helm chart values `workloads..enabled=false` when installing the traffic-manager. The following are the Helm chart values to disable the workload types: diff --git a/docs/reference/rbac.md b/docs/reference/rbac.md index ba2f1263f2..6ea64f15b1 100644 --- a/docs/reference/rbac.md +++ b/docs/reference/rbac.md @@ -8,7 +8,7 @@ toc_max_heading_level: 2 The intention of this document is to provide a template for securing and limiting the permissions of Telepresence. This documentation covers the full extent of permissions necessary to administrate Telepresence components in a cluster. -There are two general categories for cluster permissions with respect to Telepresence. There are RBAC settings for a User and for an Administrator described above. The User is expected to only have the minimum cluster permissions necessary to create a Telepresence [intercept](../howtos/intercepts.md), and otherwise be unable to affect Kubernetes resources. +There are two general categories for cluster permissions with respect to Telepresence. There are RBAC settings for a User and for an Administrator described above. The User is expected to only have the minimum cluster permissions necessary to create a Telepresence [engagement](../howtos/intercepts.md), and otherwise be unable to affect Kubernetes resources. In addition to the above, there is also a consideration of how to manage Users and Groups in Kubernetes which is outside of the scope of the document. This document will use Service Accounts to assign Roles and Bindings. Other methods of RBAC administration and enforcement can be found on the [Kubernetes RBAC documentation](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) page. diff --git a/docs/reference/routing.md b/docs/reference/routing.md index 32dafa67fb..ec48e7ee5c 100644 --- a/docs/reference/routing.md +++ b/docs/reference/routing.md @@ -14,10 +14,10 @@ When requesting a connection to a host, the IP of that host must be determined. The cluster side host lookup will be performed by a traffic-agent in the connected namespace, or by the traffic-manager if no such agent exists. ### macOS resolver -This resolver hooks into the macOS DNS system by creating files under `/etc/resolver`. Those files correspond to some domain and contain the port number of the Telepresence resolver. Telepresence creates one such file for each of the currently mapped namespaces and `include-suffixes` option. The file `telepresence.local` contains a search path that is configured based on current intercepts so that single label names can be resolved correctly. +This resolver hooks into the macOS DNS system by creating files under `/etc/resolver`. Those files correspond to some domain and contain the port number of the Telepresence resolver. Telepresence creates one such file for each of the currently mapped namespaces and `include-suffixes` option. The file `telepresence.local` contains a search path that is configured based on currently connected namespace so that single label names can be resolved correctly. ### Linux systemd-resolved resolver -This resolver registers itself as part of telepresence's [VIF](tun-device.md) using `systemd-resolved` and uses the DBus API to configure domains and routes that corresponds to the current set of intercepts and namespaces. +This resolver registers itself as part of telepresence's [VIF](tun-device.md) using `systemd-resolved` and uses the DBus API to configure domains and routes that corresponds to the connected namespace and the namespaces managed by the Traffic Manager. ### Linux overriding resolver Linux systems that aren't configured with `systemd-resolved` will use this resolver. A Typical case is when running Telepresence [inside a docker container](inside-container.md). During initialization, the resolver will first establish a _fallback_ connection to the IP passed as `--dns`, the one configured as `local-ip` in the [local DNS configuration](config.md#dns), or the primary `nameserver` registered in `/etc/resolv.conf`. It will then use iptables to actually override that IP so that requests to it instead end up in the overriding resolver, which unless it succeeds on its own, will use the _fallback_. @@ -43,7 +43,7 @@ There are multiple reasons for doing this. One is that it is important that the ```bash curl some-host ``` -results in a http request with header `Host: some-host`. Now, if a service-mesh like Istio performs header based routing, then it will fail to find that host unless the request originates from the same namespace as the host resides in. Another reason is that the configuration of a service mesh can contain very strict rules. If the request then originates from the wrong pod, it will be denied. Only one intercept at a time can be used if there is a need to ensure that the chosen pod is exactly right. +results in a http request with header `Host: some-host`. Now, if a service-mesh like Istio performs header-based routing, then it will fail to find that host unless the request originates from the same namespace as the host resides in. Another reason is that the configuration of a service mesh can contain very strict rules. If the request then originates from the wrong pod, it will be denied. ## Recursion detection It is common that clusters used in development, such as Minikube, Minishift or k3s, run on the same host as the Telepresence client, often in a Docker container. Such clusters may have access to host network, which means that both DNS and L4 routing may be subjected to recursion. diff --git a/docs/reference/tun-device.md b/docs/reference/tun-device.md index b475544239..6bd8b29dc3 100644 --- a/docs/reference/tun-device.md +++ b/docs/reference/tun-device.md @@ -5,7 +5,7 @@ hide_table_of_contents: true # Networking through Virtual Network Interface -The Telepresence daemon process creates a Virtual Network Interface (VIF) when Telepresence connects to the cluster. The VIF ensures that the cluster's subnets are available to the workstation. It also intercepts DNS requests and forwards them to the traffic-manager which in turn forwards them to intercepted agents, if any, or performs a host lookup by itself. +The Telepresence daemon process creates a Virtual Network Interface (VIF) when Telepresence connects to the cluster. The VIF ensures that the cluster's subnets are available to the workstation. It also intercepts DNS requests and forwards them to the traffic-manager which in turn forwards them to engaged agents, if any, or performs a host lookup by itself. ### TUN-Device The VIF is a TUN-device, which means that it communicates with the workstation in terms of L3 IP-packets. The router will recognize UDP and TCP packets and tunnel their payload to the traffic-manager via its encrypted gRPC API. The traffic-manager will then establish corresponding connections in the cluster. All protocol negotiation takes place in the client because the VIF takes care of the L3 to L4 translation (i.e. the tunnel is L4, not L3). @@ -26,7 +26,7 @@ port-forward`), using only one single port. There is no need for means that the traffic-manager container can run as the default user. #### sshfs without ssh encryption -When a POD is intercepted, and its volumes are mounted on the local machine, this mount is performed by [sshfs](https://github.com/libfuse/sshfs). Telepresence will run `sshfs -o slave` which means that instead of using `ssh` to establish an encrypted communication to an `sshd`, which in turn terminates the encryption and forwards to `sftp`, the `sshfs` will talk `sftp` directly on its `stdin/stdout` pair. Telepresence tunnels that directly to an `sftp` in the agent using its already encrypted gRPC API. As a result, no `sshd` is needed in client nor in the traffic-agent, and the traffic-agent container can run as the default user. +When a POD is engaged, and its volumes are mounted on the local machine, this mount is performed by [sshfs](https://github.com/libfuse/sshfs). Telepresence will run `sshfs -o slave` which means that instead of using `ssh` to establish an encrypted communication to an `sshd`, which in turn terminates the encryption and forwards to `sftp`, the `sshfs` will talk `sftp` directly on its `stdin/stdout` pair. Telepresence tunnels that directly to an `sftp` in the agent using its already encrypted gRPC API. As a result, no `sshd` is needed in client nor in the traffic-agent, and the traffic-agent container can run as the default user. ### No Firewall rules With the VIF in place, there's no longer any need to tamper with firewalls in order to establish IP routes. The VIF makes the cluster subnets available during connect, and the kernel will perform the routing automatically. When the session ends, the kernel is also responsible for cleaning up. diff --git a/docs/reference/volume.md b/docs/reference/volume.md index 4d592f7a0e..5f7003f3dd 100644 --- a/docs/reference/volume.md +++ b/docs/reference/volume.md @@ -7,26 +7,25 @@ hide_table_of_contents: true Volume mounts are achieved using a Docker Volume plug-in and Docker volume mounts when connecting using `--docker` and using `--docker-run`. This page describes how mounts are achieved when running directly on the host. -Telepresence supports locally mounting of volumes that are mounted to your Pods. You can specify a command to run when starting the intercept, this could be a subshell or local server such as Python or Node. +Telepresence supports locally mounting of volumes that are mounted to your Pods. You can specify a command to run when starting the engagement, this could be a subshell or local server such as Python or Node. ``` -telepresence intercept --port --mount=/tmp/ -- /bin/bash +telepresence replace --mount=/tmp/ -- /bin/bash ``` -In this case, Telepresence creates the intercept, mounts the Pod's volumes to locally to `/tmp`, and starts a Bash subshell. +In this case, Telepresence replaces the remote container, mounts the Pod's volumes to locally to `/tmp`, and starts a Bash subshell. Telepresence can set a random mount point for you by using `--mount=true` instead, you can then find the mount point in the output of `telepresence list` or using the `$TELEPRESENCE_ROOT` variable. ``` -$ telepresence intercept --port --mount=true -- /bin/bash +$ telepresence replace --mount=true -- /bin/bash Using Deployment -intercepted - Intercept name : +replaced + Container name : State : ACTIVE Workload kind : Deployment Destination : 127.0.0.1: Volume Mount Point: /var/folders/cp/2r22shfd50d9ymgrw14fd23r0000gp/T/telfs-988349784 - Intercepting : all TCP connections bash-3.2$ echo $TELEPRESENCE_ROOT /var/folders/cp/2r22shfd50d9ymgrw14fd23r0000gp/T/telfs-988349784 @@ -35,7 +34,7 @@ bash-3.2$ echo $TELEPRESENCE_ROOT > [!NOTE] > `--mount=true` is the default if a mount option is not specified, use `--mount=false` to disable mounting volumes. -With either method, the code you run locally either from the subshell or from the intercept command will need to be prepended with the `$TELEPRESENCE_ROOT` environment variable to utilize the mounted volumes. +With either method, the code you run locally either from the subshell or from the `replace` command will need to be prepended with the `$TELEPRESENCE_ROOT` environment variable to utilize the mounted volumes. For example, Kubernetes mounts secrets to `/var/run/secrets/kubernetes.io` (even if no `mountPoint` for it exists in the Pod spec). Once mounted, to access these you would need to change your code to use `$TELEPRESENCE_ROOT/var/run/secrets/kubernetes.io`. diff --git a/docs/reference/vpn.md b/docs/reference/vpn.md index 76579fbbcb..23989e4419 100644 --- a/docs/reference/vpn.md +++ b/docs/reference/vpn.md @@ -77,7 +77,7 @@ flag, Telepresence will take the following actions: - The local DNS-server will translate any IP contained in a VNAT subnet to a virtual IP. - All access to a virtual IP will be translated back to its original when routed to the cluster. -- The container environment retrieved when using `ingest` or `intercept` will be mangled, so that all IPs contained +- The container environment retrieved when using `replace`, `ingest`, or `intercept` will be mangled, so that all IPs contained in VNAT subnets are replaced with corresponding virtual IPs. The `--vnat` flag can be repeated to make Telepresence translate more than one subnet. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 9f6e31822d..84c0c6106b 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -31,12 +31,12 @@ FUSE library version: 2.9.9 fuse: no mount point ``` -5. Next, try a mount (or an intercept that performs a mount). It will fail because you need to give permission to “Benjamin Fleischer” to execute a kernel extension (a pop-up appears that takes you to the system preferences). +5. Next, try a mount (or an replace/ingest/intercept that performs a mount). It will fail because you need to give permission to “Benjamin Fleischer” to execute a kernel extension (a pop-up appears that takes you to the system preferences). 6. Approve the needed permission 7. Reboot your computer. ## Volume mounts are not working on Linux -It's necessary to have `sshfs` installed in order for volume mounts to work correctly during intercepts. +It's necessary to have `sshfs` installed in order for volume mounts to work correctly when Telepresence engages with remote containers. After you've installed `sshfs`, if mounts still aren't working: 1. Uncomment `user_allow_other` in `/etc/fuse.conf` diff --git a/docs/variables.yml b/docs/variables.yml index b9c6a77960..97c94fde6d 100644 --- a/docs/variables.yml +++ b/docs/variables.yml @@ -1,2 +1,2 @@ -version: "2.20.0" -dlVersion: "v2.20.0" +version: "2.22.0" +dlVersion: "v2.22.0" From 1c934987a152f4bd5c7bcfeb7e60ee3f0f54de8c Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 8 Feb 2025 16:19:31 +0100 Subject: [PATCH 56/61] Let make docs-files target generate docs/variables.yaml from changelog. Signed-off-by: Thomas Hallgren --- build-aux/main.mk | 7 ++++++- docs/README.md | 2 +- tools/src/relnotesgen/go.mod | 5 ++++- tools/src/relnotesgen/go.sum | 2 ++ tools/src/relnotesgen/main.go | 9 ++++++++- tools/src/relnotesgen/relnotes/changelog.go | 19 +++++++++++++++++++ 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/build-aux/main.mk b/build-aux/main.mk index eba0a33200..52f21eff5c 100644 --- a/build-aux/main.mk +++ b/build-aux/main.mk @@ -123,6 +123,7 @@ generate-clean: ## (Generate) Delete generated files rm -f DEPENDENCIES.md rm -f DEPENDENCY_LICENSES.md rm -f docs/release-notes.md* + rm -f docs/README.md CHANGELOG.yml: FORCE @# Check if the version is in the x.x.x format (GA release) @@ -133,7 +134,7 @@ CHANGELOG.yml: FORCE git add CHANGELOG.yml; \ fi -docs-files: docs/README.md docs/release-notes.md docs/release-notes.mdx +docs-files: docs/README.md docs/release-notes.md docs/release-notes.mdx docs/variables.yml docs/README.md: docs/doc-links.yml $(tools/tocgen) $(tools/tocgen) --input $< > $@ @@ -147,6 +148,10 @@ docs/release-notes.mdx: CHANGELOG.yml $(tools/relnotesgen) $(tools/relnotesgen) --mdx --input $< > $@ git add $@ +docs/variables.yml: CHANGELOG.yml $(tools/relnotesgen) + $(tools/relnotesgen) --variables --input $< > $@ + git add $@ + PKG_VERSION = $(shell go list ./pkg/version) # Build: artifacts that don't get checked in to Git diff --git a/docs/README.md b/docs/README.md index 0226e5bb37..5379eb57ae 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,10 +13,10 @@ raw markdown version, more bells and whistles at [telepresence.io](https://telep - Core concepts - [The developer experience and the inner dev loop](concepts/devloop.md) - [Making the remote local: Faster feedback, collaboration and debugging](concepts/faster.md) - - [Using Telepresence with Docker](howtos/docker.md) - [Intercepts](concepts/intercepts.md) - How do I... - [Code and debug an application locally](howtos/intercepts.md) + - [Using Telepresence with Docker](howtos/docker.md) - [Work with large clusters](howtos/large-clusters.md) - [Host a cluster in Docker or a VM](howtos/cluster-in-vm.md) - Technical reference diff --git a/tools/src/relnotesgen/go.mod b/tools/src/relnotesgen/go.mod index ab7c31a059..6145c1c4fe 100644 --- a/tools/src/relnotesgen/go.mod +++ b/tools/src/relnotesgen/go.mod @@ -2,4 +2,7 @@ module github.com/telepresenceio/telepresence/tools/src/relnotesgen go 1.23.0 -require sigs.k8s.io/yaml v1.4.0 +require ( + github.com/blang/semver/v4 v4.0.0 + sigs.k8s.io/yaml v1.4.0 +) diff --git a/tools/src/relnotesgen/go.sum b/tools/src/relnotesgen/go.sum index ea2896b4b5..0d1aaac4c1 100644 --- a/tools/src/relnotesgen/go.sum +++ b/tools/src/relnotesgen/go.sum @@ -1,3 +1,5 @@ +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/tools/src/relnotesgen/main.go b/tools/src/relnotesgen/main.go index 7f5d8aba8b..9ae637f5b1 100644 --- a/tools/src/relnotesgen/main.go +++ b/tools/src/relnotesgen/main.go @@ -11,10 +11,17 @@ import ( func main() { var input string var mdx bool + var variables bool flag.StringVar(&input, "input", "", "input file") flag.BoolVar(&mdx, "mdx", false, "generate mdx") + flag.BoolVar(&variables, "variables", false, "generate mdx") flag.Parse() - err := relnotes.MakeReleaseNotes(input, mdx) + var err error + if variables { + err = relnotes.MakeVariables(input) + } else { + err = relnotes.MakeReleaseNotes(input, mdx) + } if err != nil { log.Fatal(err) } diff --git a/tools/src/relnotesgen/relnotes/changelog.go b/tools/src/relnotesgen/relnotes/changelog.go index 04869c9a54..9b9eb9972f 100644 --- a/tools/src/relnotesgen/relnotes/changelog.go +++ b/tools/src/relnotesgen/relnotes/changelog.go @@ -4,11 +4,13 @@ import ( "bufio" _ "embed" "encoding/json" + "fmt" "io" "os" "text/template" "time" + "github.com/blang/semver/v4" "sigs.k8s.io/yaml" ) @@ -94,6 +96,23 @@ func MakeReleaseNotes(input string, mdx bool) error { return wr.Flush() } +func MakeVariables(input string) error { + cl, err := readChangeLog(input) + if err != nil { + return err + } + if len(cl.Items) == 0 { + return fmt.Errorf("unable to parse release version from %q", input) + } + v := cl.Items[0].Version + sv, err := semver.Parse(v) + if err != nil { + return err + } + fmt.Printf("version: \"%s\"\ndlVersion: \"v%s\"\n", sv, sv) + return nil +} + func (t *Release) UnmarshalJSON(data []byte) error { type Alias Release err := json.Unmarshal(data, (*Alias)(t)) From f57e3c10a1883e84c27a8956f0a63ae37b4c4430 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 8 Feb 2025 18:18:13 +0100 Subject: [PATCH 57/61] Ensure that agent watchers don't receive inactivated agents. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/manager/service.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index bef2d7b907..b12de4e094 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -300,6 +300,7 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa } return false } + m := mutator.GetMap(ctx) var agents []*rpc.AgentPodInfo var agentNames []string for { @@ -311,10 +312,12 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa if !ok { return nil } - agents = make([]*rpc.AgentPodInfo, len(agm)) - agentNames = make([]string, len(agm)) - i := 0 + agents = make([]*rpc.AgentPodInfo, 0, len(agm)) + agentNames = make([]string, 0, len(agm)) for _, a := range agm { + if m.IsInactive(types.UID(a.PodUid)) { + continue + } aip, err := netip.ParseAddr(a.PodIp) if err != nil { dlog.Errorf(ctx, "error parsing agent pod ip %q: %v", a.PodIp, err) @@ -327,9 +330,8 @@ func (s *service) WatchAgentPods(session *rpc.SessionInfo, stream rpc.Manager_Wa ApiPort: a.ApiPort, } ap.Intercepted = isIntercepted(ap) - agents[i] = ap - agentNames[i] = fmt.Sprintf("%s(%s)", ap.PodName, net.IP(ap.PodIp)) - i++ + agents = append(agents, ap) + agentNames = append(agentNames, fmt.Sprintf("%s(%s)", ap.PodName, net.IP(ap.PodIp))) } case is, ok := <-interceptsCh: if !ok { @@ -409,6 +411,7 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(tunnel.Sess // creating a lastSnap with one nil entry. lastSnap := make([]*rpc.AgentInfo, 1) + m := mutator.GetMap(ctx) for { select { case snapshot, ok := <-snapshotCh: @@ -417,11 +420,14 @@ func (s *service) watchAgents(ctx context.Context, includeAgent func(tunnel.Sess dlog.Debug(ctx, "Request cancelled") return nil } + + // Sort snapshot by sessionID and discard inactive agents. agentSessionIDs := slices.Sorted(maps.Keys(snapshot)) agents := make([]*rpc.AgentInfo, 0, len(agentSessionIDs)) for _, agentSessionID := range agentSessionIDs { - if as := s.state.GetAgent(agentSessionID); as != nil { - agents = append(agents, snapshot[agentSessionID].AgentInfo) + ag := snapshot[agentSessionID] + if !m.IsInactive(types.UID(ag.PodUid)) { + agents = append(agents, ag.AgentInfo) } } if slices.EqualFunc(agents, lastSnap, infosEqual) { From 59fb3f70b84b9bb935308205a85cf3d3a0eb2f4c Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sat, 8 Feb 2025 23:44:01 +0100 Subject: [PATCH 58/61] Remove all panic recovery. Most of the panic recovery routines were added to ensure logging the stack-trace of the panic. Containers in pods seem to run with `GOTRACEBACK=""` which apparently overrides the default "single" setting. To mitigate this, the traffic-manager, traffic-agent, and agent-init containers now use the `debug.SetTraceback("single")` Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/agent/agent.go | 2 ++ cmd/traffic/cmd/agentinit/agent_init.go | 8 ++------ cmd/traffic/cmd/manager/manager.go | 2 ++ cmd/traffic/cmd/manager/mutator/agent_injector.go | 8 -------- cmd/traffic/cmd/manager/mutator/service.go | 5 ----- cmd/traffic/cmd/manager/mutator/watcher.go | 5 ----- cmd/traffic/cmd/manager/service.go | 6 ------ cmd/traffic/cmd/manager/state/intercept.go | 8 -------- .../cmd/manager/state/workload_info_watcher.go | 8 ++++---- pkg/client/cli/cmd/uninstall.go | 6 ------ pkg/client/rootd/service.go | 11 +---------- pkg/client/userd/daemon/grpc.go | 10 ---------- pkg/client/userd/trafficmgr/session.go | 7 ------- pkg/vif/device_windows.go | 7 ------- pkg/vif/tunneling_device.go | 5 ----- 15 files changed, 11 insertions(+), 87 deletions(-) diff --git a/cmd/traffic/cmd/agent/agent.go b/cmd/traffic/cmd/agent/agent.go index 70c30bd774..8ee950d66c 100644 --- a/cmd/traffic/cmd/agent/agent.go +++ b/cmd/traffic/cmd/agent/agent.go @@ -10,6 +10,7 @@ import ( "os" "os/signal" "path/filepath" + "runtime/debug" "strings" "time" @@ -132,6 +133,7 @@ func sftpServer(ctx context.Context, sftpPortCh chan<- uint16) error { } func Main(ctx context.Context, _ ...string) error { + debug.SetTraceback("single") dlog.Infof(ctx, "Traffic Agent %s", version.Version) ctx, cancel := context.WithCancel(ctx) diff --git a/cmd/traffic/cmd/agentinit/agent_init.go b/cmd/traffic/cmd/agentinit/agent_init.go index 3230e4fa49..8540e75657 100644 --- a/cmd/traffic/cmd/agentinit/agent_init.go +++ b/cmd/traffic/cmd/agentinit/agent_init.go @@ -10,13 +10,13 @@ import ( "net" "net/netip" "os" + "runtime/debug" "strconv" "strings" "github.com/coreos/go-iptables/iptables" core "k8s.io/api/core/v1" - "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" "github.com/telepresenceio/telepresence/v2/pkg/version" @@ -195,12 +195,8 @@ func findLoopback() (string, error) { // Main is the main function for the agent init container. func Main(ctx context.Context, args ...string) error { + debug.SetTraceback("single") dlog.Infof(ctx, "Traffic Agent Init %s", version.Version) - defer func() { - if r := recover(); r != nil { - dlog.Error(ctx, derror.PanicToError(r)) - } - }() cfg, err := loadConfig() if err != nil { dlog.Error(ctx, err) diff --git a/cmd/traffic/cmd/manager/manager.go b/cmd/traffic/cmd/manager/manager.go index 1a086c94a0..3f1d7ab964 100644 --- a/cmd/traffic/cmd/manager/manager.go +++ b/cmd/traffic/cmd/manager/manager.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "os" + "runtime/debug" "slices" "strconv" "sync/atomic" @@ -65,6 +66,7 @@ func Main(ctx context.Context, _ ...string) error { } func MainWithEnv(ctx context.Context) (err error) { + debug.SetTraceback("single") defer runtime.RecoverFromPanic(&err) dlog.Infof(ctx, "%s %s [uid:%d,gid:%d]", DisplayName, version.Version, os.Getuid(), os.Getgid()) diff --git a/cmd/traffic/cmd/manager/mutator/agent_injector.go b/cmd/traffic/cmd/manager/mutator/agent_injector.go index 46cbfbff18..2147243ae0 100644 --- a/cmd/traffic/cmd/manager/mutator/agent_injector.go +++ b/cmd/traffic/cmd/manager/mutator/agent_injector.go @@ -20,7 +20,6 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/strings/slices" - "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/managerutil" "github.com/telepresenceio/telepresence/v2/pkg/agentconfig" @@ -90,13 +89,6 @@ func getPod(req *admission.AdmissionRequest, isDelete bool) (*core.Pod, error) { } func (a *agentInjector) Inject(ctx context.Context, req *admission.AdmissionRequest) (p PatchOps, err error) { - defer func() { - if r := recover(); r != nil { - err = derror.PanicToError(r) - dlog.Errorf(ctx, "%+v", err) - } - }() - isDelete := req.Operation == admission.Delete if atomic.LoadInt64(&a.terminating) > 0 { dlog.Debugf(ctx, "Skipping webhook for %s.%s because the agent-injector is terminating", req.Name, req.Namespace) diff --git a/cmd/traffic/cmd/manager/mutator/service.go b/cmd/traffic/cmd/manager/mutator/service.go index 38696ed698..eea515bf2f 100644 --- a/cmd/traffic/cmd/manager/mutator/service.go +++ b/cmd/traffic/cmd/manager/mutator/service.go @@ -269,11 +269,6 @@ func isNamespaceOfInterest(ns string) bool { } func serveRequest(ctx context.Context, r *http.Request, method string, f func(ctx context.Context)) (int, error) { - defer func() { - if r := recover(); r != nil { - dlog.Errorf(ctx, "%+v", derror.PanicToError(r)) - } - }() if r.Method != method { return http.StatusMethodNotAllowed, fmt.Errorf("invalid method %s, only %s requests are allowed", r.Method, method) } diff --git a/cmd/traffic/cmd/manager/mutator/watcher.go b/cmd/traffic/cmd/manager/mutator/watcher.go index b0ad6c0a33..d1a4212789 100644 --- a/cmd/traffic/cmd/manager/mutator/watcher.go +++ b/cmd/traffic/cmd/manager/mutator/watcher.go @@ -428,11 +428,6 @@ func (c *configWatcher) Start(ctx context.Context) { } func (c *configWatcher) namespacesChangeWatcher(ctx context.Context) error { - defer func() { - if err := recover(); err != nil { - dlog.Errorf(ctx, "%+v", derror.PanicToError(err)) - } - }() sid, nsChanges := namespaces.Subscribe(ctx) defer namespaces.Unsubscribe(ctx, sid) for { diff --git a/cmd/traffic/cmd/manager/service.go b/cmd/traffic/cmd/manager/service.go index b12de4e094..402096ba1c 100644 --- a/cmd/traffic/cmd/manager/service.go +++ b/cmd/traffic/cmd/manager/service.go @@ -542,12 +542,6 @@ func (s *service) WatchIntercepts(session *rpc.SessionInfo, stream rpc.Manager_W } func (s *service) PrepareIntercept(ctx context.Context, request *rpc.CreateInterceptRequest) (pi *rpc.PreparedIntercept, err error) { - defer func() { - if r := recover(); r != nil { - err = derror.PanicToError(r) - dlog.Errorf(ctx, "%+v", err) - } - }() ctx = managerutil.WithSessionInfo(ctx, request.Session) dlog.Debugf(ctx, "Intercept name %s", request.InterceptSpec.Name) return s.state.PrepareIntercept(ctx, request) diff --git a/cmd/traffic/cmd/manager/state/intercept.go b/cmd/traffic/cmd/manager/state/intercept.go index f2f0b54190..e08d73e3b3 100644 --- a/cmd/traffic/cmd/manager/state/intercept.go +++ b/cmd/traffic/cmd/manager/state/intercept.go @@ -22,7 +22,6 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" - "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dlog" rpc "github.com/telepresenceio/telepresence/rpc/v2/manager" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/managerutil" @@ -51,13 +50,6 @@ func (s *state) PrepareIntercept( ctx context.Context, cr *rpc.CreateInterceptRequest, ) (pi *rpc.PreparedIntercept, err error) { - defer func() { - if r := recover(); r != nil { - err = derror.PanicToError(r) - dlog.Errorf(ctx, "%+v", err) - } - }() - interceptError := func(err error) (*rpc.PreparedIntercept, error) { dlog.Errorf(ctx, "PrepareIntercept error %v", err) if _, ok := status.FromError(err); ok { diff --git a/cmd/traffic/cmd/manager/state/workload_info_watcher.go b/cmd/traffic/cmd/manager/state/workload_info_watcher.go index 3febc4df6b..caeb81634d 100644 --- a/cmd/traffic/cmd/manager/state/workload_info_watcher.go +++ b/cmd/traffic/cmd/manager/state/workload_info_watcher.go @@ -255,11 +255,11 @@ func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[ wl := w.Workload if wl.AgentState != as { wl.AgentState = as - dlog.Debugf(ctx, "WorkloadInfoEvent: AgentInfo %s.%s %s %s", a.Name, a.Namespace, as, wl.State) + dlog.Debugf(ctx, "WorkloadInfoEvent: AgentInfo %s(%s).%s %s %s", a.PodName, a.PodIp, a.Namespace, as, wl.State) wf.resetTicker() } } else if wl, err := agentmap.GetWorkload(ctx, name, a.Namespace, ""); err == nil { - dlog.Debugf(ctx, "WorkloadInfoEvent: AgentInfo %s.%s %s %s", a.Name, a.Namespace, as, workload.GetWorkloadState(wl)) + dlog.Debugf(ctx, "WorkloadInfoEvent: AgentInfo %s(%s).%s %s %s", a.PodName, a.PodIp, a.Namespace, as, workload.GetWorkloadState(wl)) wf.addEvent(workload.EventTypeUpdate, wl, as, nil) } else { dlog.Debugf(ctx, "Unable to get workload %s.%s: %v", name, a.Namespace, err) @@ -290,14 +290,14 @@ func (wf *workloadInfoWatcher) handleAgentSnapshot(ctx context.Context, ais map[ } if w, ok := wf.workloadEvents[name]; ok && w.Type != rpc.WorkloadEvent_DELETED { wl := w.Workload - dlog.Debugf(ctx, "WorkloadInfoEvent: AgentInfo %s.%s %s %s", a.Name, a.Namespace, as, w.Workload.State) + dlog.Debugf(ctx, "WorkloadInfoEvent: AgentInfo %s(%s).%s %s %s", a.PodName, a.PodIp, a.Namespace, as, w.Workload.State) if wl.AgentState != as { wl.AgentState = as wl.InterceptClients = iClients wf.resetTicker() } } else if wl, err := agentmap.GetWorkload(ctx, name, a.Namespace, ""); err == nil { - dlog.Debugf(ctx, "WorkloadInfoEvent: AgentInfo %s.%s %s %s", a.Name, a.Namespace, as, workload.GetWorkloadState(wl)) + dlog.Debugf(ctx, "WorkloadInfoEvent: AgentInfo %s(%s).%s %s %s", a.PodName, a.PodIp, a.Namespace, as, workload.GetWorkloadState(wl)) wf.addEvent(workload.EventTypeUpdate, wl, as, iClients) } else { dlog.Debugf(ctx, "Unable to get workload %s.%s: %v", name, a.Namespace, err) diff --git a/pkg/client/cli/cmd/uninstall.go b/pkg/client/cli/cmd/uninstall.go index 0002165322..dcd68fff40 100644 --- a/pkg/client/cli/cmd/uninstall.go +++ b/pkg/client/cli/cmd/uninstall.go @@ -7,7 +7,6 @@ import ( "github.com/spf13/cobra" - "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/rpc/v2/connector" "github.com/telepresenceio/telepresence/v2/pkg/client/cli/ann" @@ -82,11 +81,6 @@ func (u *uninstallCommand) run(cmd *cobra.Command, args []string) error { } func validWorkloads(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - defer func() { - if r := recover(); r != nil { - dlog.Errorf(cmd.Context(), "%+v", derror.PanicToError(r)) - } - }() // Trace level is used here, because we generally don't want to log expansion attempts // in the cli.log dlog.Tracef(cmd.Context(), "toComplete = %s, args = %v", toComplete, args) diff --git a/pkg/client/rootd/service.go b/pkg/client/rootd/service.go index bff4af83c4..0cc0948f0e 100644 --- a/pkg/client/rootd/service.go +++ b/pkg/client/rootd/service.go @@ -17,7 +17,6 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" - "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dgroup" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/rpc/v2/common" @@ -382,15 +381,7 @@ func (s *Service) startSession(parentCtx context.Context, oi *rpc.NetworkConfig, return reply } -func (s *Service) serveGrpc(c context.Context, l net.Listener) (err error) { - defer func() { - // Error recovery. - if perr := derror.PanicToError(recover()); perr != nil { - err = perr - dlog.Errorf(c, "%+v", perr) - } - }() - +func (s *Service) serveGrpc(c context.Context, l net.Listener) error { var opts []grpc.ServerOption cfg := client.GetConfig(c) if mz := cfg.Grpc().MaxReceiveSize(); mz > 0 { diff --git a/pkg/client/userd/daemon/grpc.go b/pkg/client/userd/daemon/grpc.go index 33801095c8..c2c1705968 100644 --- a/pkg/client/userd/daemon/grpc.go +++ b/pkg/client/userd/daemon/grpc.go @@ -16,7 +16,6 @@ import ( "google.golang.org/protobuf/types/known/emptypb" empty "google.golang.org/protobuf/types/known/emptypb" - "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dexec" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/rpc/v2/common" @@ -32,14 +31,6 @@ import ( "github.com/telepresenceio/telepresence/v2/pkg/proc" ) -func callRecovery(c context.Context, r any, err error) error { - if perr := derror.PanicToError(r); perr != nil { - dlog.Errorf(c, "%+v", perr) - err = perr - } - return err -} - func (s *service) FuseFTPError() error { return s.fuseFTPError } @@ -57,7 +48,6 @@ func (s *service) WithSession(c context.Context, f func(context.Context, userd.S // Session context has been cancelled return status.Error(codes.Canceled, "session cancelled") } - defer func() { err = callRecovery(c, recover(), err) }() return f(s.sessionContext, s.session) } diff --git a/pkg/client/userd/trafficmgr/session.go b/pkg/client/userd/trafficmgr/session.go index 6ea2832440..6b4eeabfb9 100644 --- a/pkg/client/userd/trafficmgr/session.go +++ b/pkg/client/userd/trafficmgr/session.go @@ -30,7 +30,6 @@ import ( "sigs.k8s.io/yaml" "github.com/datawire/dlib/dcontext" - "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dgroup" "github.com/datawire/dlib/dlog" "github.com/datawire/dlib/dtime" @@ -165,12 +164,6 @@ func NewSession( cr := cri.Request() connectStart := time.Now() defer func() { - if r := recover(); r != nil { - rc = ctx - err := derror.PanicToError(r) - dlog.Errorf(ctx, "%+v", err) - info = connectError(connector.ConnectInfo_DAEMON_FAILED, err) - } if info.Error == connector.ConnectInfo_UNSPECIFIED { scout.Report(ctx, "connect", scout.Entry{ diff --git a/pkg/vif/device_windows.go b/pkg/vif/device_windows.go index 99885f56dd..d56f80137a 100644 --- a/pkg/vif/device_windows.go +++ b/pkg/vif/device_windows.go @@ -17,7 +17,6 @@ import ( "gvisor.dev/gvisor/pkg/tcpip/link/channel" "gvisor.dev/gvisor/pkg/tcpip/stack" - "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dlog" ) @@ -36,12 +35,6 @@ type device struct { } func openTun(ctx context.Context) (td *device, err error) { - defer func() { - if r := recover(); r != nil { - err = derror.PanicToError(r) - dlog.Errorf(ctx, "%+v", err) - } - }() interfaceFmt := "tel%d" ifaceNumber := 0 ifaces, err := net.Interfaces() diff --git a/pkg/vif/tunneling_device.go b/pkg/vif/tunneling_device.go index 206fc6ef43..bb7a510ad0 100644 --- a/pkg/vif/tunneling_device.go +++ b/pkg/vif/tunneling_device.go @@ -7,7 +7,6 @@ import ( "github.com/hashicorp/go-multierror" "gvisor.dev/gvisor/pkg/tcpip/stack" - "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/pkg/routing" "github.com/telepresenceio/telepresence/v2/pkg/tunnel" @@ -59,10 +58,6 @@ func (vif *TunnelingDevice) Close(ctx context.Context) error { func (vif *TunnelingDevice) Run(ctx context.Context) (err error) { defer func() { - if r := recover(); r != nil { - err = derror.PanicToError(r) - dlog.Errorf(ctx, "%+v", r) - } dlog.Debug(ctx, "vif ended") }() From c5dad045a8bfe6f34d9799fc01188a001855bf1d Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Sun, 9 Feb 2025 14:49:32 +0100 Subject: [PATCH 59/61] Fixup and enable Test_LargeFiles for CI. Signed-off-by: Thomas Hallgren --- integration_test/large_files_test.go | 48 ++++++++----------- integration_test/mounts_test.go | 3 ++ .../testdata/k8s/hello-pv-volume.goyaml | 17 ++----- integration_test/testdata/k8s/local-pvc.yaml | 10 ++++ 4 files changed, 37 insertions(+), 41 deletions(-) create mode 100644 integration_test/testdata/k8s/local-pvc.yaml diff --git a/integration_test/large_files_test.go b/integration_test/large_files_test.go index 2c5a981a13..790d3cf1f7 100644 --- a/integration_test/large_files_test.go +++ b/integration_test/large_files_test.go @@ -28,7 +28,7 @@ type largeFilesSuite struct { itest.Suite itest.TrafficManager name string - manifest []byte + manifests [][]byte serviceCount int mountPoint []string largeFiles []string @@ -66,35 +66,43 @@ func (s *largeFilesSuite) ServiceCount() int { } func (s *largeFilesSuite) SetupSuite() { - if s.IsCI() || s.LargeFileTestDisabled() { - s.T().Skip("Disabled. Test is too demanding for the current setup (requires more CPU or a remote cluster)") - return - } s.Suite.SetupSuite() ctx := s.Context() - require := s.Require() wg := sync.WaitGroup{} wg.Add(s.ServiceCount()) - k8s := filepath.Join("testdata", "k8s") + mfPath := filepath.Join("testdata", "k8s", "hello-pv-volume.goyaml") + s.NoError(s.Kubectl(ctx, "apply", "-f", filepath.Join("testdata", "k8s", "local-pvc.yaml"))) + s.manifests = make([][]byte, s.ServiceCount()) for i := 0; i < s.ServiceCount(); i++ { go func(i int) { defer wg.Done() svc := fmt.Sprintf("%s-%d", s.Name(), i) - mf, err := itest.ReadTemplate(ctx, filepath.Join(k8s, "hello-pv-volume.goyaml"), &itest.PersistentVolume{ + mf, err := itest.ReadTemplate(ctx, mfPath, &itest.PersistentVolume{ Name: svc, MountDirectory: "/home/scratch", }) - s.manifest = mf - require.NoError(err) - require.NoError(s.Kubectl(dos.WithStdin(ctx, bytes.NewReader(s.manifest)), "apply", "-f", "-")) + s.NoError(err) + s.NoError(s.Kubectl(dos.WithStdin(ctx, bytes.NewReader(mf)), "apply", "-f", "-")) + s.manifests[i] = mf s.NoError(itest.RolloutStatusWait(ctx, s.AppNamespace(), "deploy/"+svc)) }(i) } wg.Wait() } -func (s *largeFilesSuite) TeardownSuite() { - s.NoError(s.Kubectl(dos.WithStdin(s.Context(), bytes.NewReader(s.manifest)), "delete", "-f", "-")) +func (s *largeFilesSuite) TearDownSuite() { + ctx := s.Context() + wg := sync.WaitGroup{} + wg.Add(s.ServiceCount()) + for i := 0; i < s.ServiceCount(); i++ { + go func(i int) { + defer wg.Done() + s.NoError(s.Kubectl(dos.WithStdin(ctx, bytes.NewReader(s.manifests[i])), "delete", "-f", "-")) + }(i) + } + s.NoError(s.Kubectl(ctx, "delete", "-f", filepath.Join("testdata", "k8s", "local-pvc.yaml"))) + itest.TelepresenceQuitOk(ctx) + wg.Wait() } func (s *largeFilesSuite) createIntercepts(ctx context.Context) { @@ -132,20 +140,6 @@ func (s *largeFilesSuite) leaveIntercepts(ctx context.Context) { } } -func (s *largeFilesSuite) TearDownSuite() { - ctx := s.Context() - wg := sync.WaitGroup{} - wg.Add(s.ServiceCount()) - for i := 0; i < s.ServiceCount(); i++ { - go func(i int) { - defer wg.Done() - s.DeleteSvcAndWorkload(ctx, "deploy", fmt.Sprintf("%s-%d", s.Name(), i)) - }(i) - } - itest.TelepresenceQuitOk(ctx) - wg.Wait() -} - func (s *largeFilesSuite) Test_LargeFileIntercepts_fuseftp() { ctx := itest.WithConfig(s.Context(), func(cfg client.Config) { cfg.Timeouts().PrivateFtpReadWrite = 2 * time.Minute diff --git a/integration_test/mounts_test.go b/integration_test/mounts_test.go index e9257c82f0..25472082fb 100644 --- a/integration_test/mounts_test.go +++ b/integration_test/mounts_test.go @@ -109,6 +109,8 @@ func (s *mountsSuite) Test_MountWrite() { ctx := s.Context() k8s := filepath.Join("testdata", "k8s") rq := s.Require() + pvcPath := filepath.Join(k8s, "local-pvc.yaml") + rq.NoError(s.Kubectl(ctx, "apply", "-f", pvcPath)) mf, err := itest.ReadTemplate(ctx, filepath.Join(k8s, "hello-pv-volume.goyaml"), &itest.PersistentVolume{ Name: "hello", MountDirectory: "/data", @@ -118,6 +120,7 @@ func (s *mountsSuite) Test_MountWrite() { rq.NoError(s.Kubectl(dos.WithStdin(ctx, bytes.NewReader(mf)), "apply", "-f", "-")) defer func() { rq.NoError(s.Kubectl(dos.WithStdin(ctx, bytes.NewReader(mf)), "delete", "-f", "-")) + rq.NoError(s.Kubectl(ctx, "delete", "-f", pvcPath)) }() mountPoint := filepath.Join(s.T().TempDir(), "mnt") diff --git a/integration_test/testdata/k8s/hello-pv-volume.goyaml b/integration_test/testdata/k8s/hello-pv-volume.goyaml index ce3c8c9c89..d9a0a5510c 100644 --- a/integration_test/testdata/k8s/hello-pv-volume.goyaml +++ b/integration_test/testdata/k8s/hello-pv-volume.goyaml @@ -3,28 +3,17 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: local-pv + name: {{ .Name }}-pv labels: purpose: tp-cli-testing spec: capacity: - storage: 0.5Gi + storage: 0.2Gi accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Retain hostPath: - path: /data/local-pv ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: local-pvc -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi + path: /data/{{ .Name }}-pv --- apiVersion: v1 kind: Service diff --git a/integration_test/testdata/k8s/local-pvc.yaml b/integration_test/testdata/k8s/local-pvc.yaml new file mode 100644 index 0000000000..ff0f760425 --- /dev/null +++ b/integration_test/testdata/k8s/local-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: local-pvc +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi From 75db6a4f2a8ba75d0272b64169d6ee2dc418b74d Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Mon, 10 Feb 2025 23:18:05 +0100 Subject: [PATCH 60/61] Rename docs/references/intercepts to engagements. Signed-off-by: Thomas Hallgren --- CHANGELOG.yml | 6 +++--- docs/README.md | 10 +++++----- docs/doc-links.yml | 10 +++++----- docs/reference/{intercepts => engagements}/cli.md | 0 .../reference/{intercepts => engagements}/container.md | 0 docs/reference/{intercepts => engagements}/sidecar.md | 0 docs/release-notes.md | 6 +++--- docs/release-notes.mdx | 6 +++--- 8 files changed, 19 insertions(+), 19 deletions(-) rename docs/reference/{intercepts => engagements}/cli.md (100%) rename docs/reference/{intercepts => engagements}/container.md (100%) rename docs/reference/{intercepts => engagements}/sidecar.md (100%) diff --git a/CHANGELOG.yml b/CHANGELOG.yml index 09c7359f7b..0590aad8a8 100644 --- a/CHANGELOG.yml +++ b/CHANGELOG.yml @@ -209,7 +209,7 @@ items: guarantees that the environment variables and mounts propagated to the client originate from the specified container. Additionally, if the `--replace` option is used, it ensures that this container is replaced. - docs: reference/intercepts/container + docs: reference/engagements/container - type: feature title: New telepresence ingest command body: >- @@ -274,7 +274,7 @@ items: The Helm chart value `workloads` now supports the kinds `deployments.enabled`, `statefulSets.enabled`, `replicaSets.enabled`. and `rollouts.enabled`. All except `rollouts` are enabled by default. The traffic-manager will ignore workloads, and Telepresence will not be able to intercept them, if the `enabled` of the corresponding kind is set to `false`. - docs: reference/intercepts/sidecar#disable-workloads + docs: reference/engagements/sidecar#disable-workloads - type: feature title: Improved command auto-completion body: >- @@ -448,7 +448,7 @@ items: instead of a service port. The new behavior is enabled by adding a telepresence.getambassador.io/inject-container-ports annotation where the value is a comma separated list of port identifiers consisting of either the name or the port number of a container port, optionally suffixed with `/TCP` or `/UDP`. - docs: https://telepresence.io/docs/reference/intercepts/cli#intercepting-without-a-service + docs: https://telepresence.io/docs/reference/engagements/cli#intercepting-without-a-service - type: feature title: Publish the OSS version of the telepresence Helm chart body: >- diff --git a/docs/README.md b/docs/README.md index 5379eb57ae..356e9dbb78 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,13 +24,13 @@ raw markdown version, more bells and whistles at [telepresence.io](https://telep - [Client reference](reference/client.md) - [Laptop-side configuration](reference/config.md) - [Cluster-side configuration](reference/cluster-config.md) - - [Using Docker for intercepts](reference/docker-run.md) + - [Using Docker for engagements](reference/docker-run.md) - [Running Telepresence in a Docker container](reference/inside-container.md) - [Environment variables](reference/environment.md) - - Intercepts - - [Configure intercept using CLI](reference/intercepts/cli.md) - - [Traffic Agent Sidecar](reference/intercepts/sidecar.md) - - [Target a specific container](reference/intercepts/container.md) + - Engagements + - [Configure intercept using CLI](reference/engagements/cli.md) + - [Traffic Agent Sidecar](reference/engagements/sidecar.md) + - [Target a specific container](reference/engagements/container.md) - [Volume mounts](reference/volume.md) - [DNS resolution](reference/dns.md) - [RBAC](reference/rbac.md) diff --git a/docs/doc-links.yml b/docs/doc-links.yml index f43ebff81b..2b1d7b0497 100644 --- a/docs/doc-links.yml +++ b/docs/doc-links.yml @@ -38,20 +38,20 @@ link: reference/config - title: Cluster-side configuration link: reference/cluster-config - - title: Using Docker for intercepts + - title: Using Docker for engagements link: reference/docker-run - title: Running Telepresence in a Docker container link: reference/inside-container - title: Environment variables link: reference/environment - - title: Intercepts + - title: Engagements items: - title: Configure intercept using CLI - link: reference/intercepts/cli + link: reference/engagements/cli - title: Traffic Agent Sidecar - link: reference/intercepts/sidecar + link: reference/engagements/sidecar - title: Target a specific container - link: reference/intercepts/container + link: reference/engagements/container - title: Volume mounts link: reference/volume - title: DNS resolution diff --git a/docs/reference/intercepts/cli.md b/docs/reference/engagements/cli.md similarity index 100% rename from docs/reference/intercepts/cli.md rename to docs/reference/engagements/cli.md diff --git a/docs/reference/intercepts/container.md b/docs/reference/engagements/container.md similarity index 100% rename from docs/reference/intercepts/container.md rename to docs/reference/engagements/container.md diff --git a/docs/reference/intercepts/sidecar.md b/docs/reference/engagements/sidecar.md similarity index 100% rename from docs/reference/intercepts/sidecar.md rename to docs/reference/engagements/sidecar.md diff --git a/docs/release-notes.md b/docs/release-notes.md index a62d931e47..6028399fb0 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -172,7 +172,7 @@ Telepresence not only detects subnet conflicts between the cluster and workstati It is now possible to use a virtual subnet without routing the affected IPs to a specific workload. A new `telepresence connect --vnat CIDR` flag was added that will perform virtual network address translation of cluster IPs. This flag is very similar to the `--proxy-via CIDR=WORKLOAD` introduced in 2.19, but without the need to specify a workload.
-##
feature
[Intercepts targeting a specific container](reference/intercepts/container)
+##
feature
[Intercepts targeting a specific container](reference/engagements/container)
In certain scenarios, the container owning the intercepted port differs @@ -233,7 +233,7 @@ See [Streaming Transitions from SPDY to WebSockets](https://kubernetes.io/blog/2 The OSS code-base will no longer report usage data to the proprietary collector at Ambassador Labs. The actual calls to the collector remain, but will be no-ops unless a proper collector client is installed using an extension point.
-##
feature
[Add deployments, statefulSets, replicaSets to workloads Helm chart value](reference/intercepts/sidecar#disable-workloads)
+##
feature
[Add deployments, statefulSets, replicaSets to workloads Helm chart value](reference/engagements/sidecar#disable-workloads)
The Helm chart value `workloads` now supports the kinds `deployments.enabled`, `statefulSets.enabled`, `replicaSets.enabled`. and `rollouts.enabled`. All except `rollouts` are enabled by default. The traffic-manager will ignore workloads, and Telepresence will not be able to intercept them, if the `enabled` of the corresponding kind is set to `false`. @@ -381,7 +381,7 @@ The name of the `telepresence gather-logs` flag `--daemons` suggests that the ar Telepresence is now capable of easily find telepresence gather-logs by certain timestamp.
-##
feature
[Enable intercepts of workloads that have no service.](https://telepresence.io/docs/reference/intercepts/cli#intercepting-without-a-service)
+##
feature
[Enable intercepts of workloads that have no service.](https://telepresence.io/docs/reference/engagements/cli#intercepting-without-a-service)
Telepresence is now capable of intercepting workloads that have no associated service. The intercept will then target container port instead of a service port. The new behavior is enabled by adding a telepresence.getambassador.io/inject-container-ports annotation where the value is a comma separated list of port identifiers consisting of either the name or the port number of a container port, optionally suffixed with `/TCP` or `/UDP`. diff --git a/docs/release-notes.mdx b/docs/release-notes.mdx index 6bcd43ede5..38c16b9393 100644 --- a/docs/release-notes.mdx +++ b/docs/release-notes.mdx @@ -137,7 +137,7 @@ namespaceSelector: It is now possible to use a virtual subnet without routing the affected IPs to a specific workload. A new `telepresence connect --vnat CIDR` flag was added that will perform virtual network address translation of cluster IPs. This flag is very similar to the `--proxy-via CIDR=WORKLOAD` introduced in 2.19, but without the need to specify a workload. - Intercepts targeting a specific container + Intercepts targeting a specific container In certain scenarios, the container owning the intercepted port differs from the container the intercept targets. This port owner's sole purpose is to route traffic from the service to the intended container, often @@ -182,7 +182,7 @@ See [Streaming Transitions from SPDY to WebSockets](https://kubernetes.io/blog/2 The OSS code-base will no longer report usage data to the proprietary collector at Ambassador Labs. The actual calls to the collector remain, but will be no-ops unless a proper collector client is installed using an extension point. - Add deployments, statefulSets, replicaSets to workloads Helm chart value + Add deployments, statefulSets, replicaSets to workloads Helm chart value The Helm chart value `workloads` now supports the kinds `deployments.enabled`, `statefulSets.enabled`, `replicaSets.enabled`. and `rollouts.enabled`. All except `rollouts` are enabled by default. The traffic-manager will ignore workloads, and Telepresence will not be able to intercept them, if the `enabled` of the corresponding kind is set to `false`. @@ -284,7 +284,7 @@ If a user should require the pod-subnet to be mapped, it can be added to the `cl Telepresence is now capable of easily find telepresence gather-logs by certain timestamp. - Enable intercepts of workloads that have no service. + Enable intercepts of workloads that have no service. Telepresence is now capable of intercepting workloads that have no associated service. The intercept will then target container port instead of a service port. The new behavior is enabled by adding a telepresence.getambassador.io/inject-container-ports annotation where the value is a comma separated list of port identifiers consisting of either the name or the port number of a container port, optionally suffixed with `/TCP` or `/UDP`. From b1fe5a6299666bee3d3f3e789eb2ba943b911747 Mon Sep 17 00:00:00 2001 From: Thomas Hallgren Date: Mon, 10 Feb 2025 23:47:59 +0100 Subject: [PATCH 61/61] Changes in response to code review. - Remove panic recovery in mutator/service.go - Remove extraneous loglevel setting in succesfulIntercept itest - Remove unused `--workload` flag from some genyaml commands. Signed-off-by: Thomas Hallgren --- cmd/traffic/cmd/manager/mutator/service.go | 7 -- integration_test/workloads_test.go | 1 - pkg/client/cli/cmd/genyaml.go | 128 +++++++-------------- 3 files changed, 42 insertions(+), 94 deletions(-) diff --git a/cmd/traffic/cmd/manager/mutator/service.go b/cmd/traffic/cmd/manager/mutator/service.go index eea515bf2f..e8b58aa15d 100644 --- a/cmd/traffic/cmd/manager/mutator/service.go +++ b/cmd/traffic/cmd/manager/mutator/service.go @@ -24,7 +24,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "github.com/datawire/dlib/dcontext" - "github.com/datawire/dlib/derror" "github.com/datawire/dlib/dgroup" "github.com/datawire/dlib/dlog" "github.com/telepresenceio/telepresence/v2/cmd/traffic/cmd/manager/managerutil" @@ -278,12 +277,6 @@ func serveRequest(ctx context.Context, r *http.Request, method string, f func(ct // serveMutatingFunc is a helper function to call a mutatorFunc. func serveMutatingFunc(ctx context.Context, r *http.Request, mf mutatorFunc) ([]byte, int, error) { - defer func() { - if r := recover(); r != nil { - dlog.Errorf(ctx, "%+v", derror.PanicToError(r)) - } - }() - // Request validations. // Only handle POST requests with a body and json content type. if r.Method != http.MethodPost { diff --git a/integration_test/workloads_test.go b/integration_test/workloads_test.go index 53dc6cbf87..db9eff6aaa 100644 --- a/integration_test/workloads_test.go +++ b/integration_test/workloads_test.go @@ -24,7 +24,6 @@ func (s *connectedSuite) successfulIntercept(tp, wl, port string) { 2*time.Second, // polling interval ) - itest.TelepresenceOk(ctx, "loglevel", "trace") stdout := itest.TelepresenceOk(ctx, "intercept", "--mount", "false", "--port", port, wl) require.Contains(stdout, "Using "+tp+" "+wl) stdout = itest.TelepresenceOk(ctx, "list", "--intercepts") diff --git a/pkg/client/cli/cmd/genyaml.go b/pkg/client/cli/cmd/genyaml.go index 37c6dde1be..40fd2ff9cb 100644 --- a/pkg/client/cli/cmd/genyaml.go +++ b/pkg/client/cli/cmd/genyaml.go @@ -52,8 +52,8 @@ NOTE: It is recommended that you not do this unless strictly necessary. Instead, telepresence's webhook injector configure the traffic agents on demand.`, ValidArgsFunction: cobra.NoFileCompletions, } - flags := cmd.PersistentFlags() - flags.StringVarP(&info.outputFile, "output", "o", "-", + fs := cmd.PersistentFlags() + fs.StringVarP(&info.outputFile, "output", "o", "-", "Path to the file to place the output in. Defaults to '-' which means stdout.") cmd.AddCommand( genConfigMapSubCommand(&info), @@ -94,38 +94,17 @@ func (i *genYAMLCommand) getOutputWriter() (io.WriteCloser, error) { return f, nil } -func (i *genYAMLCommand) loadConfigMapEntry(ctx context.Context) (*agentconfig.Sidecar, error) { - if i.configFile != "" { - b, err := getInput(i.configFile) - if err != nil { - return nil, err - } - var cfg agentconfig.Sidecar - if err = yaml.Unmarshal(b, &cfg); err != nil { - return nil, errcat.User.Newf("unable to parse config %s: %w", i.configFile, err) - } - return &cfg, nil - } - if i.workloadName == "" { - return nil, errcat.User.New("either --config or --workload must be provided") +func (i *genYAMLCommand) loadConfigMapEntry() (*agentconfig.Sidecar, error) { + if i.configFile == "" { + return nil, errcat.User.New("--config must be provided") } - - // Load configmap entry from the telepresence-agents configmap - cm, err := k8sapi.GetK8sInterface(ctx).CoreV1().ConfigMaps(i.namespace).Get(ctx, agentconfig.ConfigMap, meta.GetOptions{}) + b, err := getInput(i.configFile) if err != nil { - return nil, errcat.User.New(err) - } - var yml string - ok := false - if cm.Data != nil { - yml, ok = cm.Data[i.workloadName] - } - if !ok { - return nil, errcat.User.Newf("Unable to load entry for %q in configmap %q: %w", i.workloadName, agentconfig.ConfigMap, err) + return nil, err } var cfg agentconfig.Sidecar - if err = yaml.Unmarshal([]byte(yml), &cfg); err != nil { - return nil, errcat.User.Newf("Unable to parse entry for %q in configmap %q: %w", i.workloadName, agentconfig.ConfigMap, err) + if err = yaml.Unmarshal(b, &cfg); err != nil { + return nil, errcat.User.Newf("unable to parse config %s: %w", i.configFile, err) } return &cfg, nil } @@ -187,10 +166,10 @@ func (i *genYAMLCommand) writeObjToOutput(obj any) error { func (i *genYAMLCommand) WithJoinedClientSetInterface(ctx context.Context, flagMap map[string]string) (context.Context, error) { configFlags := genericclioptions.NewConfigFlags(false) - flags := pflag.NewFlagSet("", 0) - configFlags.AddFlags(flags) + fs := pflag.NewFlagSet("", 0) + configFlags.AddFlags(fs) for k, v := range flagMap { - if err := flags.Set(k, v); err != nil { + if err := fs.Set(k, v); err != nil { return nil, errcat.User.Newf("error processing kubectl flag --%s=%s: %w", k, v, err) } } @@ -259,22 +238,22 @@ func genConfigMapSubCommand(yamlInfo *genYAMLCommand) *cobra.Command { }, } - flags := cmd.Flags() - flags.StringVarP(&info.inputFile, "input", "i", "", + fs := cmd.Flags() + fs.StringVarP(&info.inputFile, "input", "i", "", "Path to the yaml containing the workload definition (i.e. Deployment, StatefulSet, etc). Pass '-' for stdin.. Mutually exclusive to --workload") - flags.StringVarP(&info.workloadName, "workload", "w", "", + fs.StringVarP(&info.workloadName, "workload", "w", "", "Name of the workload. If given, the workload will be retrieved from the cluster, mutually exclusive to --input") - flags.Uint16Var(&info.AgentPort, "agent-port", 9900, + fs.Uint16Var(&info.AgentPort, "agent-port", 9900, "The port number you wish the agent to listen on.") - flags.StringVar(&info.QualifiedAgentImage, "agent-image", "ghcr.io/telepresenceio/tel2:"+strings.TrimPrefix(client.Version(), "v"), + fs.StringVar(&info.QualifiedAgentImage, "agent-image", "ghcr.io/telepresenceio/tel2:"+strings.TrimPrefix(client.Version(), "v"), `The qualified name of the agent image`) - flags.Uint16Var(&info.ManagerPort, "manager-port", 8081, + fs.Uint16Var(&info.ManagerPort, "manager-port", 8081, `The traffic-manager API port`) - flags.StringVar(&info.ManagerNamespace, "manager-namespace", "ambassador", + fs.StringVar(&info.ManagerNamespace, "manager-namespace", "ambassador", `The traffic-manager namespace`) - flags.StringVar(&info.LogLevel, "loglevel", "info", + fs.StringVar(&info.LogLevel, "loglevel", "info", `The loglevel for the generated traffic-agent sidecar`) - flags.AddFlagSet(kubeFlags) + fs.AddFlagSet(kubeFlags) return cmd } @@ -321,13 +300,11 @@ func genContainerSubCommand(yamlInfo *genYAMLCommand) *cobra.Command { return info.run(cmd, flags.Map(kubeFlags)) }, } - flags := cmd.Flags() - flags.StringVarP(&info.inputFile, "input", "i", "", + fs := cmd.Flags() + fs.StringVarP(&info.inputFile, "input", "i", "", "Optional path to the yaml containing the workload definition (i.e. Deployment, StatefulSet, etc). Pass '-' for stdin. Loaded from cluster by default") - flags.StringVarP(&info.workloadName, "workload", "w", "", - "Name of the workload. If given, the configmap entry will be retrieved telepresence-agents configmap, mutually exclusive to --config") - flags.StringVarP(&info.configFile, "config", "c", "", "Path to the yaml containing the generated configmap entry, mutually exclusive to --workload") - flags.AddFlagSet(kubeFlags) + fs.StringVarP(&info.configFile, "config", "c", "", "Path to the yaml containing the generated configmap entry") + fs.AddFlagSet(kubeFlags) return cmd } @@ -337,7 +314,7 @@ func (g *genContainerInfo) run(cmd *cobra.Command, kubeFlags map[string]string) return err } - cm, err := g.loadConfigMapEntry(ctx) + cm, err := g.loadConfigMapEntry() if err != nil { return err } @@ -390,21 +367,14 @@ func genInitContainerSubCommand(yamlInfo *genYAMLCommand) *cobra.Command { return info.run(cmd, flags.Map(kubeFlags)) }, } - flags := cmd.Flags() - flags.StringVarP(&info.workloadName, "workload", "w", "", - "Name of the workload. If given, the configmap entry will be retrieved telepresence-agents configmap, mutually exclusive to --config") - flags.StringVarP(&info.configFile, "config", "c", "", "Path to the yaml containing the generated configmap entry, mutually exclusive to --workload") - flags.AddFlagSet(kubeFlags) + fs := cmd.Flags() + fs.StringVarP(&info.configFile, "config", "c", "", "Path to the yaml containing the generated configmap entry") + fs.AddFlagSet(kubeFlags) return cmd } -func (g *genInitContainerInfo) run(cmd *cobra.Command, kubeFlags map[string]string) error { - ctx, err := g.WithJoinedClientSetInterface(cmd.Context(), kubeFlags) - if err != nil { - return err - } - - cm, err := g.loadConfigMapEntry(ctx) +func (g *genInitContainerInfo) run(*cobra.Command, map[string]string) error { + cm, err := g.loadConfigMapEntry() if err != nil { return err } @@ -425,34 +395,22 @@ type genAnnotationsInfo struct { func genVAnnotationsSubCommand(yamlInfo *genYAMLCommand) *cobra.Command { info := genAnnotationsInfo{genYAMLCommand: yamlInfo} - kubeFlags := allKubeFlags() cmd := &cobra.Command{ Use: "annotations", Args: cobra.NoArgs, Short: "Generate YAML for the pod template metadata annotations.", Long: "Generate YAML for the pod template metadata annotations. See genyaml for more info on what this means", - RunE: func(cmd *cobra.Command, args []string) error { - return info.run(cmd, flags.Map(kubeFlags)) + RunE: func(*cobra.Command, []string) error { + return info.run() }, } - flags := cmd.Flags() - flags.StringVarP(&info.workloadName, "workload", "w", "", - "Name of the workload. If given, the configmap entry will be retrieved telepresence-agents configmap, mutually exclusive to --config") - flags.StringVarP(&info.configFile, "config", "c", "", "Path to the yaml containing the generated configmap entry, mutually exclusive to --workload") - flags.AddFlagSet(kubeFlags) + fs := cmd.Flags() + fs.StringVarP(&info.configFile, "config", "c", "", "Path to the yaml containing the generated configmap entry") return cmd } -func (g *genAnnotationsInfo) run(cmd *cobra.Command, kubeFlags map[string]string) error { - ctx := cmd.Context() - if g.configFile == "" { - var err error - ctx, err = g.WithJoinedClientSetInterface(ctx, kubeFlags) - if err != nil { - return err - } - } - cm, err := g.loadConfigMapEntry(ctx) +func (g *genAnnotationsInfo) run() error { + cm, err := g.loadConfigMapEntry() if err != nil { return err } @@ -484,13 +442,11 @@ func genVolumeSubCommand(yamlInfo *genYAMLCommand) *cobra.Command { return info.run(cmd, flags.Map(kubeFlags)) }, } - flags := cmd.Flags() - flags.StringVarP(&info.inputFile, "input", "i", "", + fs := cmd.Flags() + fs.StringVarP(&info.inputFile, "input", "i", "", "Optional path to the yaml containing the workload definition (i.e. Deployment, StatefulSet, etc). Pass '-' for stdin. Loaded from cluster by default") - flags.StringVarP(&info.workloadName, "workload", "w", "", - "Name of the workload. If given, the configmap entry will be retrieved telepresence-agents configmap, mutually exclusive to --config") - flags.StringVarP(&info.configFile, "config", "c", "", "Path to the yaml containing the generated configmap entry, mutually exclusive to --workload") - flags.AddFlagSet(kubeFlags) + fs.StringVarP(&info.configFile, "config", "c", "", "Path to the yaml containing the generated configmap entry") + fs.AddFlagSet(kubeFlags) return cmd } @@ -500,7 +456,7 @@ func (g *genVolumeInfo) run(cmd *cobra.Command, kubeFlags map[string]string) err return err } - cm, err := g.loadConfigMapEntry(ctx) + cm, err := g.loadConfigMapEntry() if err != nil { return err }