feat: add usearch

Makefile

@@ -85,6 +85,7 @@ BUF_VERSION               := $(eval BUF_VERSION := $(shell cat versions/BUF_VERS
 CMAKE_VERSION             := $(eval CMAKE_VERSION := $(shell cat versions/CMAKE_VERSION))$(CMAKE_VERSION)
 DOCKER_VERSION            := $(eval DOCKER_VERSION := $(shell cat versions/DOCKER_VERSION))$(DOCKER_VERSION)
 FAISS_VERSION             := $(eval FAISS_VERSION := $(shell cat versions/FAISS_VERSION))$(FAISS_VERSION)
+USEARCH_VERSION           := $(eval USEARCH_VERSION := $(shell cat versions/USEARCH_VERSION))$(USEARCH_VERSION)
 GOLANGCILINT_VERSION      := $(eval GOLANGCILINT_VERSION := $(shell cat versions/GOLANGCILINT_VERSION))$(GOLANGCILINT_VERSION)
 GO_VERSION                := $(eval GO_VERSION := $(shell cat versions/GO_VERSION))$(GO_VERSION)
 HDF5_VERSION              := $(eval HDF5_VERSION := $(shell cat versions/HDF5_VERSION))$(HDF5_VERSION)
@@ -603,6 +604,11 @@ version/ngt:
 version/faiss:
 	@echo $(FAISS_VERSION)
 
+.PHONY: version/usearch
+## print usearch version
+version/usearch:
+	@echo $(USEARCH_VERSION)
+
 .PHONY: version/docker
 ## print Kubernetes version
 version/docker:
@@ -677,6 +683,22 @@ $(LIB_PATH)/libfaiss.a:
 	rm -rf $(TEMP_DIR)/v$(FAISS_VERSION).tar.gz $(TEMP_DIR)/faiss-$(FAISS_VERSION)
 	ldconfig
 
+.PHONY: usearch/install
+## install usearch
+usearch/install: 
+ifeq ($(OS),linux)
+	curl -sSL https://github.com/unum-cloud/usearch/releases/download/v$(USEARCH_VERSION)/usearch_$(OS)_$(GOARCH)_$(USEARCH_VERSION).deb -o usearch_$(OS)_$(USEARCH_VERSION).deb
+	dpkg -i usearch_$(OS)_$(USEARCH_VERSION).deb
+	rm usearch_$(OS)_$(USEARCH_VERSION).deb
+	ldconfig
+else ifeq ($(OS),macos)
+	curl -sSL https://github.com/unum-cloud/usearch/releases/download/v$(USEARCH_VERSION)/usearch_macos_$(GOARCH)_$(USEARCH_VERSION).zip -o usearch_macos_$(OS)_$(USEARCH_VERSION).zip
+	unzip usearch_macos_$(OS)_$(USEARCH_VERSION).zip
+	sudo mv libusearch_c.dylib /usr/local/lib && sudo mv usearch.h /usr/local/include
+	rm -rf usearch_macos_$(OS)_$(USEARCH_VERSION).zip
+	ldconfig
+endif
+
 .PHONY: cmake/install
 ## install CMAKE
 cmake/install:

dockers/ci/base/Dockerfile

@@ -120,6 +120,7 @@ RUN --mount=type=bind,target=.,rw \
     && make telepresence/install \
     && make ngt/install \
     && make faiss/install \
+    && make usearch/install \
     && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
 # skipcq: DOK-DL3002
 USER root:root

go.mod

@@ -380,6 +380,7 @@ require (
 	github.com/quasilyte/go-ruleguard/dsl v0.3.22
 	github.com/scylladb/gocqlx v0.0.0-00010101000000-000000000000
 	github.com/stretchr/testify v1.9.0
+	github.com/unum-cloud/usearch/golang v0.0.0-20240828190432-b9a9758a06e1
 	github.com/zeebo/xxh3 v1.0.2
 	go.etcd.io/bbolt v1.3.8
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0

go.sum

@@ -635,6 +635,8 @@ github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vl
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+github.com/unum-cloud/usearch/golang v0.0.0-20240828190432-b9a9758a06e1 h1:hILse+Dt0Sk6RfyG19Ld48kcdTOnHx2F6dm3QH1X4Mw=
+github.com/unum-cloud/usearch/golang v0.0.0-20240828190432-b9a9758a06e1/go.mod h1:NxBpQibuBBeA/V8RGbrNzVAv4OyWWL5yNao7mVz656k=
 github.com/urfave/cli/v2 v2.4.0/go.mod h1:NX9W0zmTvedE5oDoOMs2RTC8RvdK98NTYZE5LbaEYPg=
 github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
 github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=

hack/actions/gen/main.go

@@ -313,6 +313,7 @@ const (
 	rustVersionPath        = versionsPath + "/RUST_VERSION"
 	faissVersionPath       = versionsPath + "/FAISS_VERSION"
 	ngtVersionPath         = versionsPath + "/NGT_VERSION"
+	usearchVersionPath     = versionsPath + "/USEARCH_VERSION"
 
 	makefilePath    = "Makefile"
 	makefileDirPath = "Makefile.d/**"

hack/docker/gen/main.go

@@ -241,8 +241,9 @@
 
 	agentInernalPackage = "pkg/agent/internal"
 
-	ngtPreprocess   = "make ngt/install"
-	faissPreprocess = "make faiss/install"
+	ngtPreprocess     = "make ngt/install"
+	faissPreprocess   = "make faiss/install"
+	usearchPreprocess = "make usearch/install"
 
 	helmOperatorRootdir   = "/opt/helm"
 	helmOperatorWatchFile = helmOperatorRootdir + "/watches.yaml"
@@ -645,7 +646,7 @@
 				append(ngtBuildDeps,
 					append(faissBuildDeps,
 						devContainerDeps...)...)...)...),
-			Preprocess:  append(ciContainerPreprocess, ngtPreprocess, faissPreprocess),
+			Preprocess:  append(ciContainerPreprocess, ngtPreprocess, faissPreprocess, usearchPreprocess),
 			Entrypoints: []string{"/bin/bash"},
 		},
 		"vald-dev-container": {
@@ -663,7 +664,8 @@
 			Preprocess: append(devContainerPreprocess,
 				append(ciContainerPreprocess,
 					ngtPreprocess,
-					faissPreprocess)...),
+					faissPreprocess,
+					usearchPreprocess)...),
 		},
 		"vald-buildbase": {
 			AppName:      "buildbase",

internal/core/algorithm/usearch/option.go

@@ -0,0 +1,164 @@
+//
+// Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package usearch provides implementation of Go API for https://github.com/unum-cloud/usearch
+package usearch
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/kpango/fastime"
+	core "github.com/unum-cloud/usearch/golang"
+	"github.com/vdaas/vald/internal/core/algorithm"
+	"github.com/vdaas/vald/internal/errors"
+)
+
+// Option represents the functional option for usearch.
+type Option func(*usearch) error
+
+var defaultOptions = []Option{
+	WithIndexPath("/tmp/usearch-" + strconv.FormatInt(fastime.UnixNanoNow(), 10)),
+	WithQuantizationType("F32"),
+	WithMetricType("cosine"),
+	WithDimension(64),
+	WithConnectivity(0),
+	WithExpansionAdd(0),
+	WithExpansionSearch(0),
+	WithMulti(false),
+}
+
+// WithIndexPath represents the option to set the index path for usearch.
+func WithIndexPath(path string) Option {
+	return func(u *usearch) error {
+		if len(path) == 0 {
+			return errors.NewErrIgnoredOption("indexPath")
+		}
+		u.idxPath = path
+		return nil
+	}
+}
+
+// WithQuantizationType represents the option to set the quantizationType for usearch.
+func WithQuantizationType(quantizationType string) Option {
+	return func(u *usearch) error {
+		switch quantizationType {
+		case "BF16":
+			u.quantizationType = core.BF16
+		case "F16":
+			u.quantizationType = core.F16
+		case "F32":
+			u.quantizationType = core.F32
+		case "F64":
+			u.quantizationType = core.F64
+		case "I8":
+			u.quantizationType = core.I8
+		case "B1":
+			u.quantizationType = core.B1
+		default:
+			err := errors.NewUsearchError("unsupported QuantizationType")
+			return errors.NewErrCriticalOption("QuantizationType", quantizationType, err)
+		}
+		return nil
+	}
+}
+
+// WithMetricType represents the option to set the metricType for usearch.
+func WithMetricType(metricType string) Option {
+	return func(u *usearch) error {
+		switch strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(metricType)) {
+		case "l2sq":
+			u.metricType = core.L2sq
+		case "ip":
+			u.metricType = core.InnerProduct
+		case "cosine":
+			u.metricType = core.Cosine
+		case "haversine":
+			u.metricType = core.Haversine
+		case "divergence":
+			u.metricType = core.Divergence
+		case "pearson":
+			u.metricType = core.Pearson
+		case "hamming":
+			u.metricType = core.Hamming
+		case "tanimoto":
+			u.metricType = core.Tanimoto
+		case "sorensen":
+			u.metricType = core.Sorensen
+		default:
+			err := errors.NewUsearchError("unsupported MetricType")
+			return errors.NewErrCriticalOption("MetricType", metricType, err)
+		}
+		return nil
+	}
+}
+
+// WithDimension represents the option to set the dimension for usearch.
+func WithDimension(dim int) Option {
+	return func(u *usearch) error {
+		if dim > algorithm.MaximumVectorDimensionSize || dim < algorithm.MinimumVectorDimensionSize {
+			err := errors.ErrInvalidDimensionSize(dim, algorithm.MaximumVectorDimensionSize)
+			return errors.NewErrCriticalOption("dimension", dim, err)
+		}
+
+		u.dimension = uint(dim)
+		return nil
+	}
+}
+
+// WithConnectivity represents the option to set the connectivity for usearch.
+func WithConnectivity(connectivity int) Option {
+	return func(u *usearch) error {
+		if connectivity < 0 {
+			return errors.NewErrInvalidOption("Connectivity", connectivity)
+		}
+
+		u.connectivity = uint(connectivity)
+		return nil
+	}
+}
+
+// WithExpansionAdd represents the option to set the expansion add for usearch.
+func WithExpansionAdd(expansionAdd int) Option {
+	return func(u *usearch) error {
+		if expansionAdd < 0 {
+			return errors.NewErrInvalidOption("Expansion Add", expansionAdd)
+		}
+
+		u.expansionAdd = uint(expansionAdd)
+		return nil
+	}
+}
+
+// WithExpansionSearch represents the option to set the expansion search for usearch.
+func WithExpansionSearch(expansionSearch int) Option {
+	return func(u *usearch) error {
+		if expansionSearch < 0 {
+			return errors.NewErrInvalidOption("Expansion Search", expansionSearch)
+		}
+
+		u.expansionSearch = uint(expansionSearch)
+		return nil
+	}
+}
+
+// WithMulti represents the option to set the multi for usearch.
+func WithMulti(multi bool) Option {
+	return func(u *usearch) error {
+		u.multi = multi
+		return nil
+	}
+}