Refctor for release v1.7.14

vdaas · Sep 23, 2024 · 985f0e5 · 985f0e5
1 parent bcebbdc
commit 985f0e5
Show file tree

Hide file tree

Showing 67 changed files with 873 additions and 352 deletions.
diff --git a/.gitfiles b/.gitfiles
@@ -167,6 +167,8 @@ apis/grpc/v1/filter/egress/egress_filter.pb.go
 apis/grpc/v1/filter/egress/egress_filter_vtproto.pb.go
 apis/grpc/v1/filter/ingress/ingress_filter.pb.go
 apis/grpc/v1/filter/ingress/ingress_filter_vtproto.pb.go
+apis/grpc/v1/meta/meta.pb.go
+apis/grpc/v1/meta/meta_vtproto.pb.go
 apis/grpc/v1/mirror/mirror.go
 apis/grpc/v1/mirror/mirror.pb.go
 apis/grpc/v1/mirror/mirror_vtproto.pb.go
@@ -203,6 +205,7 @@ apis/proto/v1/agent/sidecar/sidecar.proto
 apis/proto/v1/discoverer/discoverer.proto
 apis/proto/v1/filter/egress/egress_filter.proto
 apis/proto/v1/filter/ingress/ingress_filter.proto
+apis/proto/v1/meta/meta.proto
 apis/proto/v1/mirror/mirror.proto
 apis/proto/v1/payload/payload.proto
 apis/proto/v1/rpc/errdetails/error_details.proto
@@ -220,6 +223,7 @@ apis/swagger/v1/agent/sidecar/sidecar.swagger.json
 apis/swagger/v1/discoverer/discoverer.swagger.json
 apis/swagger/v1/filter/egress/egress_filter.swagger.json
 apis/swagger/v1/filter/ingress/ingress_filter.swagger.json
+apis/swagger/v1/meta/meta.swagger.json
 apis/swagger/v1/mirror/mirror.swagger.json
 apis/swagger/v1/payload/payload.swagger.json
 apis/swagger/v1/rpc/errdetails/error_details.swagger.json
@@ -882,6 +886,9 @@ internal/core/algorithm/ngt/ngt_bench_test.go
 internal/core/algorithm/ngt/ngt_test.go
 internal/core/algorithm/ngt/option.go
 internal/core/algorithm/ngt/option_test.go
+internal/core/algorithm/usearch/option.go
+internal/core/algorithm/usearch/usearch.go
+internal/core/algorithm/usearch/usearch_test.go
 internal/db/kvs/bbolt/bbolt.go
 internal/db/kvs/bbolt/bbolt_test.go
 internal/db/kvs/bbolt/option.go
@@ -1024,6 +1031,7 @@ internal/errors/storage.go
 internal/errors/tls.go
 internal/errors/unit.go
 internal/errors/unit_test.go
+internal/errors/usearch.go
 internal/errors/vald.go
 internal/errors/vald_test.go
 internal/errors/vqueue.go
@@ -1911,6 +1919,10 @@ rust/bin/agent/src/handler/search.rs
 rust/bin/agent/src/handler/update.rs
 rust/bin/agent/src/handler/upsert.rs
 rust/bin/agent/src/main.rs
+rust/bin/meta/Cargo.toml
+rust/bin/meta/src/handler.rs
+rust/bin/meta/src/handler/meta.rs
+rust/bin/meta/src/main.rs
 rust/libs/algorithm/Cargo.toml
 rust/libs/algorithm/src/lib.rs
 rust/libs/algorithms/faiss/Cargo.toml
@@ -1931,6 +1943,7 @@ rust/libs/proto/src/discoverer.v1.tonic.rs
 rust/libs/proto/src/filter.egress.v1.tonic.rs
 rust/libs/proto/src/filter.ingress.v1.tonic.rs
 rust/libs/proto/src/lib.rs
+rust/libs/proto/src/meta.v1.tonic.rs
 rust/libs/proto/src/mirror.v1.tonic.rs
 rust/libs/proto/src/payload.v1.rs
 rust/libs/proto/src/rpc.v1.rs
@@ -1986,6 +1999,7 @@ versions/PROTOBUF_VERSION
 versions/REVIEWDOG_VERSION
 versions/RUST_VERSION
 versions/TELEPRESENCE_VERSION
+versions/USEARCH_VERSION
 versions/VALD_VERSION
 versions/YQ_VERSION
 versions/ZLIB_VERSION

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -25,8 +25,8 @@ assignees: ""
 - Vald Version: v1.7.13
 - Go Version: v1.23.1
 - Rust Version: v1.81.0
-- Docker Version: v27.2.1
-- Kubernetes Version: v1.31.0
-- Helm Version: v3.16.0
+- Docker Version: v27.3.1
+- Kubernetes Version: v1.31.1
+- Helm Version: v3.16.1
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0
diff --git a/.github/ISSUE_TEMPLATE/security_issue_report.md b/.github/ISSUE_TEMPLATE/security_issue_report.md
@@ -19,8 +19,8 @@ assignees: ""
 - Vald Version: v1.7.13
 - Go Version: v1.23.1
 - Rust Version: v1.81.0
-- Docker Version: v27.2.1
-- Kubernetes Version: v1.31.0
-- Helm Version: v3.16.0
+- Docker Version: v27.3.1
+- Kubernetes Version: v1.31.1
+- Helm Version: v3.16.1
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -3,7 +3,7 @@
 ### Description
 
 <!-- Describe your changes in detail -->
-<!-- It would be better to describe the details especially What changed and Why you changed -->
+<!-- It would be better to describe the details, especially What changed and Why you changed -->
 
 ### Related Issue
 
@@ -18,9 +18,9 @@
 - Vald Version: v1.7.13
 - Go Version: v1.23.1
 - Rust Version: v1.81.0
-- Docker Version: v27.2.1
-- Kubernetes Version: v1.31.0
-- Helm Version: v3.16.0
+- Docker Version: v27.3.1
+- Kubernetes Version: v1.31.1
+- Helm Version: v3.16.1
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0
 
@@ -34,4 +34,4 @@
 
 ### Special notes for your reviewer
 
-<!-- Please tell us anything you would like to share to reviewers related this PR -->
+<!-- Please tell us anything you would like to share with reviewers related to this PR. Your thoughts and feedback are highly valued -->
diff --git a/Makefile b/Makefile
@@ -665,7 +665,7 @@ $(LIB_PATH)/libfaiss.a:
 		-DBUILD_STATIC_EXECS=ON \
 		-DBUILD_TESTING=OFF \
 		-DFAISS_ENABLE_PYTHON=OFF \
-	        -DFAISS_ENABLE_GPU=OFF \
+		-DFAISS_ENABLE_GPU=OFF \
 		-DBLA_VENDOR=OpenBLAS \
 		-DCMAKE_C_FLAGS="$(LDFLAGS)" \
 		-DCMAKE_EXE_LINKER_FLAGS="$(FAISS_LDFLAGS)" \
@@ -679,19 +679,29 @@ $(LIB_PATH)/libfaiss.a:
 
 .PHONY: usearch/install
 ## install usearch
-usearch/install: 
-ifeq ($(OS),linux)
-	curl -sSL https://github.com/unum-cloud/usearch/releases/download/v$(USEARCH_VERSION)/usearch_$(OS)_$(GOARCH)_$(USEARCH_VERSION).deb -o usearch_$(OS)_$(USEARCH_VERSION).deb
-	dpkg -i usearch_$(OS)_$(USEARCH_VERSION).deb
-	rm usearch_$(OS)_$(USEARCH_VERSION).deb
-	ldconfig
-else ifeq ($(OS),macos)
-	curl -sSL https://github.com/unum-cloud/usearch/releases/download/v$(USEARCH_VERSION)/usearch_macos_$(GOARCH)_$(USEARCH_VERSION).zip -o usearch_macos_$(OS)_$(USEARCH_VERSION).zip
-	unzip usearch_macos_$(OS)_$(USEARCH_VERSION).zip
-	sudo mv libusearch_c.dylib /usr/local/lib && sudo mv usearch.h /usr/local/include
-	rm -rf usearch_macos_$(OS)_$(USEARCH_VERSION).zip
+usearch/install:
+	git clone --depth 1 --recursive --branch v$(USEARCH_VERSION) https://github.com/unum-cloud/usearch $(TEMP_DIR)/usearch-$(USEARCH_VERSION)
+	cd $(TEMP_DIR)/usearch-$(USEARCH_VERSION) && \
+	cmake -DCMAKE_BUILD_TYPE=Release \
+		-DBUILD_SHARED_LIBS=OFF \
+		-DBUILD_TESTING=OFF \
+		-DUSEARCH_BUILD_LIB_C=ON \
+		-DUSEARCH_USE_FP16LIB=ON \
+		-DUSEARCH_USE_OPENMP=ON \
+		-DUSEARCH_USE_SIMSIMD=ON \
+		-DUSEARCH_USE_JEMALLOC=ON \
+		-DCMAKE_C_FLAGS="$(CFLAGS)" \
+		-DCMAKE_CXX_FLAGS="$(CXXFLAGS)" \
+		-DCMAKE_INSTALL_PREFIX=$(USR_LOCAL) \
+		-DCMAKE_INSTALL_LIBDIR=$(LIB_PATH) \
+		-B $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build $(TEMP_DIR)/usearch-$(USEARCH_VERSION)
+	cmake --build $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build -j$(CORES)
+	cmake --install $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build --prefix=$(USR_LOCAL)
+	cd $(ROOTDIR)
+	cp $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build/libusearch_static_c.a $(LIB_PATH)/libusearch.a
+	cp $(TEMP_DIR)/usearch-$(USEARCH_VERSION)/build/libusearch_c.so $(LIB_PATH)/libusearch.so
+	rm -rf $(TEMP_DIR)/usearch-$(USEARCH_VERSION)
 	ldconfig
-endif
 
 .PHONY: cmake/install
 ## install CMAKE
@@ -710,7 +720,6 @@ cmake/install:
 	cd $(ROOTDIR)
 	rm -rf $(TEMP_DIR)/CMAKE-$(CMAKE_VERSION)
 	ldconfig
-	# -DCMAKE_USE_OPENSSL=OFF
 
 .PHONY: lint
 ## run lints

diff --git a/apis/grpc/v1/agent/sidecar/sidecar_vtproto.pb.go b/apis/grpc/v1/agent/sidecar/sidecar_vtproto.pb.go
@@ -36,7 +36,7 @@ const _ = grpc.SupportPackageIsVersion7
 // SidecarClient is the client API for Sidecar service.
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
-type SidecarClient interface{}
+type SidecarClient any
 
 type sidecarClient struct {
 	cc grpc.ClientConnInterface

diff --git a/apis/grpc/v1/payload/payload.pb.go b/apis/grpc/v1/payload/payload.pb.go
diff --git a/apis/grpc/v1/rpc/errdetails/error_details.pb.go b/apis/grpc/v1/rpc/errdetails/error_details.pb.go