Skip to content

Commit

Permalink
Merge branch 'main' into feat/implement-delete-expired-index
Browse files Browse the repository at this point in the history
  • Loading branch information
vankichi authored Oct 24, 2024
2 parents 3bdb7cf + 22195f1 commit ab8671f
Show file tree
Hide file tree
Showing 24 changed files with 257 additions and 19 deletions.
76 changes: 76 additions & 0 deletions .github/workflows/dockers-example-client-image.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#
# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go

name: "Build docker image: example-client"
on:
push:
branches:
- main
- release/v*.*
- "!release/v*.*.*"
tags:
- "*.*.*"
- v*.*.*
- "*.*.*-*"
- v*.*.*-*
pull_request:
paths:
- hack/docker/gen/main.go
- dockers/example/client/Dockerfile
- hack/actions/gen/main.go
- .github/workflows/dockers-example-client-image.yaml
- .github/actions/docker-build/action.yaml
- .github/workflows/_docker-image.yaml
- example/client/**
- apis/grpc/**
- apis/proto/**
- go.mod
- go.sum
- versions/GO_VERSION
- internal/**
- "!internal/**/*_test.go"
- "!internal/**/*_mock.go"
- "!internal/db/**"
- Makefile
- Makefile.d/**
pull_request_target:
paths:
- hack/docker/gen/main.go
- dockers/example/client/Dockerfile
- hack/actions/gen/main.go
- .github/workflows/dockers-example-client-image.yaml
- .github/actions/docker-build/action.yaml
- .github/workflows/_docker-image.yaml
- example/client/**
- apis/grpc/**
- apis/proto/**
- go.mod
- go.sum
- versions/GO_VERSION
- internal/**
- "!internal/**/*_test.go"
- "!internal/**/*_mock.go"
- "!internal/db/**"
- Makefile
- Makefile.d/**
jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
with:
target: example-client
secrets: inherit
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ TAG ?= latest
CRORG ?= $(ORG)
GHCRORG = ghcr.io/$(REPO)
AGENT_IMAGE = $(NAME)-agent
AGENT_NGT_IMAGE = $(AGENT_IMAGE)-ngt
AGENT_FAISS_IMAGE = $(AGENT_IMAGE)-faiss
AGENT_NGT_IMAGE = $(AGENT_IMAGE)-ngt
AGENT_SIDECAR_IMAGE = $(AGENT_IMAGE)-sidecar
BENCHMARK_JOB_IMAGE = $(NAME)-benchmark-job
BENCHMARK_OPERATOR_IMAGE = $(NAME)-benchmark-operator
Expand All @@ -36,6 +36,7 @@ BUILDKIT_SYFT_SCANNER_IMAGE = $(BUILDKIT_IMAGE)-syft-scanner
CI_CONTAINER_IMAGE = $(NAME)-ci-container
DEV_CONTAINER_IMAGE = $(NAME)-dev-container
DISCOVERER_IMAGE = $(NAME)-discoverer-k8s
EXAMPLE_CLIENT_IMAGE = $(NAME)-example-client
FILTER_GATEWAY_IMAGE = $(NAME)-filter-gateway
HELM_OPERATOR_IMAGE = $(NAME)-helm-operator
INDEX_CORRECTION_IMAGE = $(NAME)-index-correction
Expand Down
10 changes: 10 additions & 0 deletions Makefile.d/build.mk
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ binary/build: \
cmd/tools/benchmark/job/job \
cmd/tools/benchmark/operator/operator \
cmd/tools/cli/loadtest/loadtest \
example/client/client \
cmd/agent/core/ngt/ngt \
cmd/agent/core/faiss/faiss \
rust/target/debug/agent \
Expand Down Expand Up @@ -108,6 +109,10 @@ cmd/tools/cli/loadtest/loadtest:
$(eval CGO_ENABLED = 1)
$(call go-build,tools/cli/loadtest,-linkmode 'external',$(LDFLAGS) $(HDF5_LDFLAGS), cgo,$(HDF5_VERSION),$@)

example/client/client:
$(eval CGO_ENABLED = 1)
$(call go-example-build,example/client,-linkmode 'external',$(LDFLAGS) $(HDF5_LDFLAGS), cgo,$(HDF5_VERSION),$@)

rust/target/release/agent:
pushd rust && cargo build -p agent --release && popd

Expand All @@ -124,6 +129,7 @@ binary/build/zip: \
artifacts/vald-benchmark-operator-$(GOOS)-$(GOARCH).zip \
artifacts/vald-cli-loadtest-$(GOOS)-$(GOARCH).zip \
artifacts/vald-discoverer-k8s-$(GOOS)-$(GOARCH).zip \
artifacts/vald-example-client-$(GOOS)-$(GOARCH).zip \
artifacts/vald-filter-gateway-$(GOOS)-$(GOARCH).zip \
artifacts/vald-index-correction-$(GOOS)-$(GOARCH).zip \
artifacts/vald-index-creation-$(GOOS)-$(GOARCH).zip \
Expand Down Expand Up @@ -202,3 +208,7 @@ artifacts/vald-readreplica-rotate-$(GOOS)-$(GOARCH).zip: cmd/index/job/readrepli
artifacts/vald-index-operator-$(GOOS)-$(GOARCH).zip: cmd/index/operator/index-operator
$(call mkdir, $(dir $@))
zip --junk-paths $@ $<

artifacts/vald-example-client-$(GOOS)-$(GOARCH).zip: example/client/client
$(call mkdir, $(dir $@))
zip --junk-paths $@ $<
16 changes: 15 additions & 1 deletion Makefile.d/docker.mk
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,18 @@ docker/build: \
docker/build/ci-container \
docker/build/dev-container \
docker/build/discoverer-k8s \
docker/build/example-client \
docker/build/gateway-filter \
docker/build/gateway-lb \
docker/build/gateway-mirror \
docker/build/helm-operator \
docker/build/index-correction \
docker/build/index-creation \
docker/build/index-deletion \
docker/build/index-operator \
docker/build/index-save \
docker/build/loadtest \
docker/build/manager-index \
docker/build/helm-operator \
docker/build/readreplica-rotate

docker/xpanes/build:
Expand All @@ -57,6 +58,7 @@ docker/xpanes/build:
docker/build/ci-container \
docker/build/dev-container \
docker/build/discoverer-k8s \
docker/build/example-client \
docker/build/gateway-filter \
docker/build/gateway-lb \
docker/build/gateway-mirror \
Expand Down Expand Up @@ -396,3 +398,15 @@ docker/build/benchmark-operator:
@make DOCKERFILE="$(ROOTDIR)/dockers/tools/benchmark/operator/Dockerfile" \
IMAGE=$(BENCHMARK_OPERATOR_IMAGE) \
docker/build/image

.PHONY: docker/name/example-client
docker/name/example-client:
@echo "$(ORG)/$(EXAMPLE_CLIENT_IMAGE)"

.PHONY: docker/build/example-client
## build example client docker image
docker/build/example-client:
@make DOCKERFILE="$(ROOTDIR)/dockers/example/client/Dockerfile" \
IMAGE=$(EXAMPLE_CLIENT_IMAGE) \
DOCKER_OPTS="--build-arg ZLIB_VERSION=$(ZLIB_VERSION) --build-arg HDF5_VERSION=$(HDF5_VERSION)" \
docker/build/image
29 changes: 29 additions & 0 deletions Makefile.d/functions.mk
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,35 @@ define go-build
$6 -version
endef

define go-example-build
echo $(GO_SOURCES_INTERNAL)
echo $(PBGOS)
echo $(shell find $(ROOTDIR)/$1 -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
cd $(ROOTDIR)/$1 && \
CFLAGS="$(CFLAGS)" \
CXXFLAGS="$(CXXFLAGS)" \
CGO_ENABLED=$(CGO_ENABLED) \
CGO_CXXFLAGS="$3" \
CGO_FFLAGS="$3" \
CGO_LDFLAGS="$3" \
GO111MODULE=on \
GOARCH=$(GOARCH) \
GOOS=$(GOOS) \
GOPRIVATE=$(GOPRIVATE) \
GO_VERSION=$(GO_VERSION) \
go build \
--ldflags "-w $2 \
-extldflags '$3' \
-buildid=" \
-modcacherw \
-mod=readonly \
-a \
-tags "osusergo netgo static_build$4" \
-trimpath \
-o $(ROOTDIR)/$6 \
main.go
endef

define telepresence
[ -z $(SWAP_IMAGE) ] && IMAGE=$2 || IMAGE=$(SWAP_IMAGE) \
&& echo "telepresence replaces $(SWAP_DEPLOYMENT_TYPE)/$1 with $${IMAGE}:$(SWAP_TAG)" \
Expand Down
95 changes: 95 additions & 0 deletions dockers/example/client/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
# syntax = docker/dockerfile:latest
#
# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
ARG UPX_OPTIONS=-9
# skipcq: DOK-DL3026,DOK-DL3007
FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder
LABEL maintainer="vdaas.org vald team <[email protected]>"
# skipcq: DOK-DL3002
USER root:root
ARG TARGETARCH
ARG TARGETOS
ARG GO_VERSION
ARG RUST_VERSION
ENV APP_NAME=client
ENV DEBIAN_FRONTEND=noninteractive
ENV GO111MODULE=on
ENV GOPATH=/go
ENV GOROOT=/opt/go
ENV HOME=/root
ENV INITRD=No
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US.UTF-8
ENV LC_ALL=en_US.UTF-8
ENV ORG=vdaas
ENV PKG=example/client
ENV REPO=vald
ENV TZ=Etc/UTC
ENV USER=root
ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH}
WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008
RUN --mount=type=bind,target=.,rw \
--mount=type=tmpfs,target=/tmp \
--mount=type=cache,target=/var/lib/apt,sharing=locked,id=${APP_NAME} \
--mount=type=cache,target=/var/cache/apt,sharing=locked,id=${APP_NAME} \
--mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \
--mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \
--mount=type=tmpfs,target="${GOPATH}/src" \
set -ex \
&& echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
&& echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \
&& apt-get clean \
&& apt-get update -y \
&& apt-get upgrade -y \
&& apt-get install -y --no-install-recommends --fix-missing \
build-essential \
ca-certificates \
curl \
tzdata \
locales \
git \
cmake \
g++ \
gcc \
libssl-dev \
unzip \
libhdf5-dev \
libaec-dev \
&& ldconfig \
&& echo "${LANG} UTF-8" > /etc/locale.gen \
&& ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \
&& locale-gen ${LANGUAGE} \
&& update-locale LANG=${LANGUAGE} \
&& dpkg-reconfigure -f noninteractive tzdata \
&& apt-get clean \
&& apt-get autoclean -y \
&& apt-get autoremove -y \
&& make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/install \
&& make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/download \
&& make hdf5/install \
&& make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" ${PKG}/${APP_NAME} \
&& mv "${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
# skipcq: DOK-DL3026,DOK-DL3007
FROM gcr.io/distroless/static:nonroot
LABEL maintainer="vdaas.org vald team <[email protected]>"
COPY --from=builder /usr/bin/client /usr/bin/client
# skipcq: DOK-DL3002
USER nonroot:nonroot
ENTRYPOINT ["/usr/bin/client"]
1 change: 0 additions & 1 deletion docs/user-guides/sdks.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ Here is the list of Official SDKs.
- [Java](https://github.com/vdaas/vald-client-java)
- [Python](https://github.com/vdaas/vald-client-python)
- [Node.js](https://github.com/vdaas/vald-client-node)
- [Clojure](https://github.com/vdaas/vald-client-clj)

## How to use SDKs

Expand Down
2 changes: 1 addition & 1 deletion example/helm/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ agent:
# The number of dimensions for feature vector of fashion-mnist dataset.
dimension: 784
# We use L2-Norm for distance_type.
distance_type: cos
distance_type: l2
# The type of fashion-mnist's feature vectors.
object_type: float
# Check duration of automatic indexing.
Expand Down
14 changes: 14 additions & 0 deletions hack/docker/gen/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,10 @@ var (
"make GOARCH=\"${TARGETARCH}\" GOOS=\"${TARGETOS}\" REPO=\"${ORG}\" NAME=\"${REPO}\" cmd/${PKG}/${APP_NAME}",
"mv \"cmd/${PKG}/${APP_NAME}\" \"{{$.BinDir}}/${APP_NAME}\"",
}
goExampleBuildCommands = []string{
"make GOARCH=\"${TARGETARCH}\" GOOS=\"${TARGETOS}\" REPO=\"${ORG}\" NAME=\"${REPO}\" ${PKG}/${APP_NAME}",
"mv \"${PKG}/${APP_NAME}\" \"{{$.BinDir}}/${APP_NAME}\"",
}
rustBuildCommands = []string{
"make rust/target/release/${APP_NAME}",
"mv \"rust/target/release/${APP_NAME}\" \"{{$.BinDir}}/${APP_NAME}\"",
Expand Down Expand Up @@ -711,6 +715,14 @@ func main() {
ngtPreprocess,
faissPreprocess)...),
},
"vald-example-client": {
AppName: "client",
PackageDir: "example/client",
ExtraPackages: append(clangBuildDeps, "libhdf5-dev", "libaec-dev"),
Preprocess: []string{
"make hdf5/install",
},
},
"vald-buildbase": {
AppName: "buildbase",
AliasImage: true,
Expand Down Expand Up @@ -788,6 +800,8 @@ func main() {
}
if file.Exists(file.Join(os.Args[1], "cmd", data.PackageDir)) {
commands = append(commands, goBuildCommands...)
} else if strings.HasPrefix(data.PackageDir, "example") && file.Exists(file.Join(os.Args[1], data.PackageDir)) {
commands = append(commands, goExampleBuildCommands...)
}
data.RunCommands = commands
mounts := make([]string, 0, len(defaultMounts)+len(goDefaultMounts))
Expand Down
2 changes: 1 addition & 1 deletion pkg/gateway/mirror/usecase/vald.go
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ func (r *run) Stop(ctx context.Context) (errs error) {
return errs
}

// PtopStop is a method called after execution of Stop.
// PostStop is a method called after execution of Stop.
func (*run) PostStop(_ context.Context) error {
return nil
}
2 changes: 1 addition & 1 deletion pkg/index/job/creation/service/indexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ func (idx *index) doCreateIndex(
switch {
case errors.Is(err, context.Canceled):
err = status.WrapWithCanceled(
agent.CreateIndexRPCName+" API canceld", err,
agent.CreateIndexRPCName+" API canceled", err,
)
attrs = trace.StatusCodeCancelled(err.Error())
case errors.Is(err, context.DeadlineExceeded):
Expand Down
2 changes: 1 addition & 1 deletion pkg/index/job/creation/usecase/creation.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ func (r *run) Stop(ctx context.Context) (errs error) {
return errs
}

// PtopStop is a method called after execution of Stop.
// PostStop is a method called after execution of Stop.
func (*run) PostStop(_ context.Context) error {
return nil
}
2 changes: 1 addition & 1 deletion pkg/index/job/readreplica/rotate/usecase/rotate.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func (r *run) Stop(ctx context.Context) (errs error) {
return errs
}

// PtopStop is a method called after execution of Stop.
// PostStop is a method called after execution of Stop.
func (*run) PostStop(_ context.Context) error {
return nil
}
Loading

0 comments on commit ab8671f

Please sign in to comment.