add efficient search logic for large top-k users

Signed-off-by: kpango <[email protected]>

.gitfiles

@@ -75,6 +75,7 @@
 .github/workflows/coverage.yml
 .github/workflows/detect-internal-config-changes.yml
 .github/workflows/dockers-agent-faiss-image.yml
+.github/workflows/dockers-agent-image.yml
 .github/workflows/dockers-agent-ngt-image.yml
 .github/workflows/dockers-agent-sidecar-image.yml
 .github/workflows/dockers-benchmark-job-image.yml
@@ -167,8 +168,10 @@ apis/grpc/v1/mirror/mirror.pb.go
 apis/grpc/v1/mirror/mirror_vtproto.pb.go
 apis/grpc/v1/payload/interface.go
 apis/grpc/v1/payload/payload.pb.go
+apis/grpc/v1/payload/payload.pb.json.go
 apis/grpc/v1/payload/payload_vtproto.pb.go
 apis/grpc/v1/rpc/errdetails/error_details.pb.go
+apis/grpc/v1/rpc/errdetails/error_details.pb.json.go
 apis/grpc/v1/rpc/errdetails/error_details_vtproto.pb.go
 apis/grpc/v1/vald/filter.pb.go
 apis/grpc/v1/vald/filter_vtproto.pb.go
@@ -243,6 +246,7 @@ assets/docs/overview/search_flow.svg
 assets/docs/overview/update_flow.svg
 assets/docs/overview/upsert_flow.svg
 assets/docs/overview/vald_basic_architecture.svg
+assets/docs/performance/benchmark-grafana.png
 assets/docs/troubleshooting/provisioning_flow_chart.drawio
 assets/docs/troubleshooting/provisioning_flow_chart.svg
 assets/docs/tutorial/getstarted.svg
@@ -505,6 +509,8 @@ design/Vald Architecture Assets.drawio
 design/Vald Architecture Dataflow.drawio
 design/Vald Architecture Overview.drawio
 design/vald_architecture_overview.png
+dockers/agent/core/agent/Dockerfile
+dockers/agent/core/agent/README.md
 dockers/agent/core/faiss/Dockerfile
 dockers/agent/core/ngt/Dockerfile
 dockers/agent/core/ngt/README.md
@@ -1121,6 +1127,7 @@ internal/net/grpc/handler.go
 internal/net/grpc/health/health.go
 internal/net/grpc/health/health_test.go
 internal/net/grpc/interceptor.go
+internal/net/grpc/interceptor/client/metric/metric.go
 internal/net/grpc/interceptor/client/trace/trace.go
 internal/net/grpc/interceptor/server/logging/accesslog.go
 internal/net/grpc/interceptor/server/logging/accesslog_test.go
@@ -1814,6 +1821,11 @@ rust/Cargo.lock
 rust/Cargo.toml
 rust/bin/agent/Cargo.toml
 rust/bin/agent/src/main.rs
+rust/libs/ngt-rs/Cargo.toml
+rust/libs/ngt-rs/build.rs
+rust/libs/ngt-rs/src/input.cpp
+rust/libs/ngt-rs/src/input.h
+rust/libs/ngt-rs/src/lib.rs
 rust/libs/ngt/Cargo.toml
 rust/libs/ngt/src/lib.rs
 rust/libs/proto/Cargo.toml
@@ -1836,6 +1848,7 @@ rust/libs/proto/src/sidecar.v1.rs
 rust/libs/proto/src/sidecar.v1.tonic.rs
 rust/libs/proto/src/vald.v1.rs
 rust/libs/proto/src/vald.v1.tonic.rs
+rust/rust-toolchain.toml
 tests/chaos/chart/.helmignore
 tests/chaos/chart/Chart.yaml
 tests/chaos/chart/README.md
@@ -1881,6 +1894,7 @@ versions/OTEL_OPERATOR_VERSION
 versions/PROMETHEUS_STACK_VERSION
 versions/PROTOBUF_VERSION
 versions/REVIEWDOG_VERSION
+versions/RUST_VERSION
 versions/TELEPRESENCE_VERSION
 versions/VALDCLI_VERSION
 versions/VALD_VERSION

.github/ISSUE_TEMPLATE/bug_report.md

@@ -25,5 +25,5 @@ assignees: ""
 - Go Version: 1.22.2
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.29.3
+- Kubernetes Version: v1.30.0
 - NGT Version: 2.2.1

.github/ISSUE_TEMPLATE/security_issue_report.md

@@ -19,5 +19,5 @@ assignees: ""
 - Go Version: 1.22.2
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.29.3
+- Kubernetes Version: v1.30.0
 - NGT Version: 2.2.1

.github/PULL_REQUEST_TEMPLATE.md

@@ -19,7 +19,7 @@
 - Go Version: 1.22.2
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.29.3
+- Kubernetes Version: v1.30.0
 - NGT Version: 2.2.1
 
 ### Checklist

apis/docs/v1/docs.md

@@ -795,17 +795,18 @@ Search related messages.
 
 Represent search configuration.
 
-| Field                 | Type                                                                   | Label | Description                              |
-| --------------------- | ---------------------------------------------------------------------- | ----- | ---------------------------------------- |
-| request_id            | [string](#string)                                                      |       | Unique request ID.                       |
-| num                   | [uint32](#uint32)                                                      |       | Maximum number of result to be returned. |
-| radius                | [float](#float)                                                        |       | Search radius.                           |
-| epsilon               | [float](#float)                                                        |       | Search coefficient.                      |
-| timeout               | [int64](#int64)                                                        |       | Search timeout in nanoseconds.           |
-| ingress_filters       | [Filter.Config](#payload-v1-Filter-Config)                             |       | Ingress filter configurations.           |
-| egress_filters        | [Filter.Config](#payload-v1-Filter-Config)                             |       | Egress filter configurations.            |
-| min_num               | [uint32](#uint32)                                                      |       | Minimum number of result to be returned. |
-| aggregation_algorithm | [Search.AggregationAlgorithm](#payload-v1-Search-AggregationAlgorithm) |       | Aggregation Algorithm                    |
+| Field                 | Type                                                                   | Label | Description                                  |
+| --------------------- | ---------------------------------------------------------------------- | ----- | -------------------------------------------- |
+| request_id            | [string](#string)                                                      |       | Unique request ID.                           |
+| num                   | [uint32](#uint32)                                                      |       | Maximum number of result to be returned.     |
+| radius                | [float](#float)                                                        |       | Search radius.                               |
+| epsilon               | [float](#float)                                                        |       | Search coefficient.                          |
+| timeout               | [int64](#int64)                                                        |       | Search timeout in nanoseconds.               |
+| ingress_filters       | [Filter.Config](#payload-v1-Filter-Config)                             |       | Ingress filter configurations.               |
+| egress_filters        | [Filter.Config](#payload-v1-Filter-Config)                             |       | Egress filter configurations.                |
+| min_num               | [uint32](#uint32)                                                      |       | Minimum number of result to be returned.     |
+| aggregation_algorithm | [Search.AggregationAlgorithm](#payload-v1-Search-AggregationAlgorithm) |       | Aggregation Algorithm                        |
+| ratio                 | [google.protobuf.FloatValue](#google-protobuf-FloatValue)              |       | Search ratio for agent return result number. |
 
 <a name="payload-v1-Search-IDRequest"></a>