feat: add usearch

Makefile

@@ -85,6 +85,7 @@ BUF_VERSION               := $(eval BUF_VERSION := $(shell cat versions/BUF_VERS
 CMAKE_VERSION             := $(eval CMAKE_VERSION := $(shell cat versions/CMAKE_VERSION))$(CMAKE_VERSION)
 DOCKER_VERSION            := $(eval DOCKER_VERSION := $(shell cat versions/DOCKER_VERSION))$(DOCKER_VERSION)
 FAISS_VERSION             := $(eval FAISS_VERSION := $(shell cat versions/FAISS_VERSION))$(FAISS_VERSION)
+USEARCH_VERSION           := $(eval USEARCH_VERSION := $(shell cat versions/USEARCH_VERSION))$(USEARCH_VERSION)
 GOLANGCILINT_VERSION      := $(eval GOLANGCILINT_VERSION := $(shell cat versions/GOLANGCILINT_VERSION))$(GOLANGCILINT_VERSION)
 GO_VERSION                := $(eval GO_VERSION := $(shell cat versions/GO_VERSION))$(GO_VERSION)
 HDF5_VERSION              := $(eval HDF5_VERSION := $(shell cat versions/HDF5_VERSION))$(HDF5_VERSION)
@@ -601,6 +602,11 @@ version/ngt:
 version/faiss:
 	@echo $(FAISS_VERSION)
 
+.PHONY: version/usearch
+## print usearch version
+version/usearch:
+	@echo $(USEARCH_VERSION)
+
 .PHONY: version/docker
 ## print Kubernetes version
 version/docker:
@@ -671,6 +677,22 @@ $(LIB_PATH)/libfaiss.a:
 	rm -rf $(TEMP_DIR)/v$(FAISS_VERSION).tar.gz $(TEMP_DIR)/faiss-$(FAISS_VERSION)
 	ldconfig
 
+.PHONY: usearch/install
+## install usearch
+usearch/install: 
+ifeq ($(OS),linux)
+	curl -sSL https://github.com/unum-cloud/usearch/releases/download/v$(USEARCH_VERSION)/usearch_$(OS)_$(GOARCH)_$(USEARCH_VERSION).deb -o usearch_$(OS)_$(USEARCH_VERSION).deb
+	dpkg -i usearch_$(OS)_$(USEARCH_VERSION).deb
+	rm usearch_$(OS)_$(USEARCH_VERSION).deb
+	ldconfig
+else ifeq ($(OS),macos)
+	curl -sSL https://github.com/unum-cloud/usearch/releases/download/v$(USEARCH_VERSION)/usearch_macos_$(GOARCH)_$(USEARCH_VERSION).zip -o usearch_macos_$(OS)_$(USEARCH_VERSION).zip
+	unzip usearch_macos_$(OS)_$(USEARCH_VERSION).zip
+	sudo mv libusearch_c.dylib /usr/local/lib && sudo mv usearch.h /usr/local/include
+	rm -rf usearch_macos_$(OS)_$(USEARCH_VERSION).zip
+	ldconfig
+endif
+
 .PHONY: cmake/install
 ## install CMAKE
 cmake/install:

dockers/ci/base/Dockerfile

@@ -121,6 +121,7 @@ RUN --mount=type=bind,target=.,rw \
     && make telepresence/install \
     && make ngt/install \
     && make faiss/install \
+    && make usearch/install \
     && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
 # skipcq: DOK-DL3002
 USER root:root

go.mod

@@ -380,6 +380,7 @@ require (
 	github.com/quasilyte/go-ruleguard/dsl v0.3.22
 	github.com/scylladb/gocqlx v0.0.0-00010101000000-000000000000
 	github.com/stretchr/testify v1.9.0
+	github.com/unum-cloud/usearch/golang v0.0.0-20240828190432-b9a9758a06e1
 	github.com/zeebo/xxh3 v1.0.2
 	go.etcd.io/bbolt v1.3.8
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0

go.sum

@@ -641,6 +641,8 @@ github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vl
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+github.com/unum-cloud/usearch/golang v0.0.0-20240828190432-b9a9758a06e1 h1:hILse+Dt0Sk6RfyG19Ld48kcdTOnHx2F6dm3QH1X4Mw=
+github.com/unum-cloud/usearch/golang v0.0.0-20240828190432-b9a9758a06e1/go.mod h1:NxBpQibuBBeA/V8RGbrNzVAv4OyWWL5yNao7mVz656k=
 github.com/urfave/cli/v2 v2.4.0/go.mod h1:NX9W0zmTvedE5oDoOMs2RTC8RvdK98NTYZE5LbaEYPg=
 github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
 github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=

hack/actions/gen/main.go

@@ -314,6 +314,7 @@ const (
 	rustVersionPath        = versionsPath + "/RUST_VERSION"
 	faissVersionPath       = versionsPath + "/FAISS_VERSION"
 	ngtVersionPath         = versionsPath + "/NGT_VERSION"
+	usearchVersionPath     = versionsPath + "/USEARCH_VERSION"
 
 	makefilePath    = "Makefile"
 	makefileDirPath = "Makefile.d/**"

hack/docker/gen/main.go

@@ -246,8 +246,9 @@
 
 	agentInernalPackage = "pkg/agent/internal"
 
-	ngtPreprocess   = "make ngt/install"
-	faissPreprocess = "make faiss/install"
+	ngtPreprocess     = "make ngt/install"
+	faissPreprocess   = "make faiss/install"
+	usearchPreprocess = "make usearch/install"
 
 	helmOperatorRootdir   = "/opt/helm"
 	helmOperatorWatchFile = helmOperatorRootdir + "/watches.yaml"
@@ -649,7 +650,7 @@
 				append(ngtBuildDeps,
 					append(faissBuildDeps,
 						devContainerDeps...)...)...)...),
-			Preprocess:  append(ciContainerPreprocess, ngtPreprocess, faissPreprocess),
+			Preprocess:  append(ciContainerPreprocess, ngtPreprocess, faissPreprocess, usearchPreprocess),
 			Entrypoints: []string{"/bin/bash"},
 		},
 		"vald-dev-container": {
@@ -667,7 +668,8 @@
 			Preprocess: append(devContainerPreprocess,
 				append(ciContainerPreprocess,
 					ngtPreprocess,
-					faissPreprocess)...),
+					faissPreprocess,
+					usearchPreprocess)...),
 		},
 		"vald-buildbase": {
 			AppName:      "buildbase",

internal/core/algorithm/usearch/option.go

@@ -0,0 +1,156 @@
+//
+// Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package usearch provides implementation of Go API for https://github.com/unum-cloud/usearch
+package usearch
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/kpango/fastime"
+	core "github.com/unum-cloud/usearch/golang"
+	"github.com/vdaas/vald/internal/core/algorithm"
+	"github.com/vdaas/vald/internal/errors"
+)
+
+// Option represents the functional option for usearch.
+type Option func(*usearch) error
+
+var defaultOptions = []Option{
+	WithIndexPath("/tmp/usearch-" + strconv.FormatInt(fastime.UnixNanoNow(), 10)),
+	WithQuantizationType("F32"),
+	WithMetricType("cosine"),
+	WithDimension(64),
+	WithConnectivity(0),
+	WithExpansionAdd(0),
+	WithExpansionSearch(0),
+	WithMulti(false),
+}
+
+// WithIndexPath represents the option to set the index path for usearch.
+func WithIndexPath(path string) Option {
+	return func(u *usearch) error {
+		if len(path) == 0 {
+			return errors.NewErrIgnoredOption("indexPath")
+		}
+		u.idxPath = path
+		return nil
+	}
+}
+
+// WithQuantizationType represents the option to set the quantizationType for usearch.
+func WithQuantizationType(quantizationType string) Option {
+	return func(u *usearch) error {
+		quantizationTypeMap := map[string]core.Quantization{
+			"BF16": core.BF16,
+			"F16":  core.F16,
+			"F32":  core.F32,
+			"F64":  core.F64,
+			"I8":   core.I8,
+			"B1":   core.B1,
+		}
+		if quantizationType, ok := quantizationTypeMap[quantizationType]; ok {
+			u.quantizationType = quantizationType
+		} else {
+			err := errors.NewUsearchError("unsupported QuantizationType")
+			return errors.NewErrCriticalOption("QuantizationType", quantizationType, err)
+		}
+		return nil
+	}
+}
+
+// WithMetricType represents the option to set the metricType for usearch.
+func WithMetricType(metricType string) Option {
+	return func(u *usearch) error {
+		metricTypeMap := map[string]core.Metric{
+			"l2sq":       core.L2sq,
+			"ip":         core.InnerProduct,
+			"cosine":     core.Cosine,
+			"haversine":  core.Haversine,
+			"divergence": core.Divergence,
+			"pearson":    core.Pearson,
+			"hamming":    core.Hamming,
+			"tanimoto":   core.Tanimoto,
+			"sorensen":   core.Sorensen,
+		}
+		normalizedMetricType := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(metricType))
+		if metricType, ok := metricTypeMap[normalizedMetricType]; ok {
+			u.metricType = metricType
+		} else {
+			err := errors.NewUsearchError("unsupported MetricType")
+			return errors.NewErrCriticalOption("MetricType", metricType, err)
+		}
+		return nil
+	}
+}
+
+// WithDimension represents the option to set the dimension for usearch.
+func WithDimension(dim int) Option {
+	return func(u *usearch) error {
+		if dim > algorithm.MaximumVectorDimensionSize || dim < algorithm.MinimumVectorDimensionSize {
+			err := errors.ErrInvalidDimensionSize(dim, algorithm.MaximumVectorDimensionSize)
+			return errors.NewErrCriticalOption("dimension", dim, err)
+		}
+
+		u.dimension = uint(dim)
+		return nil
+	}
+}
+
+// WithConnectivity represents the option to set the connectivity for usearch.
+func WithConnectivity(connectivity int) Option {
+	return func(u *usearch) error {
+		if connectivity < 0 {
+			return errors.NewErrInvalidOption("Connectivity", connectivity)
+		}
+
+		u.connectivity = uint(connectivity)
+		return nil
+	}
+}
+
+// WithExpansionAdd represents the option to set the expansion add for usearch.
+func WithExpansionAdd(expansionAdd int) Option {
+	return func(u *usearch) error {
+		if expansionAdd < 0 {
+			return errors.NewErrInvalidOption("Expansion Add", expansionAdd)
+		}
+
+		u.expansionAdd = uint(expansionAdd)
+		return nil
+	}
+}
+
+// WithExpansionSearch represents the option to set the expansion search for usearch.
+func WithExpansionSearch(expansionSearch int) Option {
+	return func(u *usearch) error {
+		if expansionSearch < 0 {
+			return errors.NewErrInvalidOption("Expansion Search", expansionSearch)
+		}
+
+		u.expansionSearch = uint(expansionSearch)
+		return nil
+	}
+}
+
+// WithMulti represents the option to set the multi for usearch.
+func WithMulti(multi bool) Option {
+	return func(u *usearch) error {
+		u.multi = multi
+		return nil
+	}
+}