Skip to content

Commit

Permalink
add pyvsag package tool (#233)
Browse files Browse the repository at this point in the history
  • Loading branch information
wxyucs authored Dec 24, 2024
1 parent bdb4716 commit b12a120
Show file tree
Hide file tree
Showing 10 changed files with 347 additions and 100 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -181,3 +181,6 @@ data/
/testresult/
*.index

/build-release/
*.so.[0-9]*
/python/pyvsag/_version.py
7 changes: 4 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -266,10 +266,11 @@ endif ()

# pybinds
if (ENABLE_PYBINDS)
find_package (Python3 REQUIRED COMPONENTS Interpreter Development)
find_package (Python3 REQUIRED COMPONENTS Interpreter Development.Module)
include (extern/pybind11/pybind11.cmake)
pybind11_add_module (pyvsag python_bindings/binding.cpp)
target_link_libraries (pyvsag PRIVATE pybind11::module vsag)
pybind11_add_module (_pyvsag python_bindings/binding.cpp)
target_compile_options (_pyvsag PRIVATE -fopenmp)
target_link_libraries (_pyvsag PRIVATE pybind11::module vsag)
endif ()

# install
Expand Down
69 changes: 41 additions & 28 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
CMAKE_GENERATOR ?= "Unix Makefiles"
CMAKE_INSTALL_PREFIX ?= "/usr/local/"
COMPILE_JOBS ?= 6
VSAG_CMAKE_ARGS = -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} -DNUM_BUILDING_JOBS=${COMPILE_JOBS} -DENABLE_TESTS=1 -DENABLE_PYBINDS=1 -G ${CMAKE_GENERATOR} -S. -Bbuild
DEBUG_BUILD_DIR ?= "./build/"
RELEASE_BUILD_DIR ?= "./build-release/"
VSAG_CMAKE_ARGS = -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} -DNUM_BUILDING_JOBS=${COMPILE_JOBS} -DENABLE_TESTS=1 -DENABLE_PYBINDS=1 -G ${CMAKE_GENERATOR} -S.
UT_FILTER = ""
ifdef CASE
UT_FILTER = $(CASE)
Expand All @@ -20,25 +22,11 @@ help: ## Show the help.
@echo "Targets:"
@fgrep "##" Makefile | fgrep -v fgrep

# ================= development part =================
.PHONY: debug
debug: ## Build vsag with debug options.
cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_CCACHE=ON -DENABLE_ASAN=OFF
cmake --build build --parallel ${COMPILE_JOBS}

.PHONY: release
release: ## Build vsag with release options.
cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Release
cmake --build build --parallel ${COMPILE_JOBS}

.PHONY: distribution
distribution: ## Build vsag with distribution options.
cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Release -DENABLE_CXX11_ABI=off -DENABLE_LIBCXX=off
cmake --build build --parallel ${COMPILE_JOBS}

.PHONY: libcxx
libcxx: ## Build vsag using libc++.
cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Release -DENABLE_LIBCXX=on
cmake --build build --parallel ${COMPILE_JOBS}
cmake ${VSAG_CMAKE_ARGS} -B${DEBUG_BUILD_DIR} -DCMAKE_BUILD_TYPE=Debug -DENABLE_CCACHE=ON -DENABLE_ASAN=OFF
cmake --build ${DEBUG_BUILD_DIR} --parallel ${COMPILE_JOBS}

.PHONY: fmt
fmt: ## Format codes.
Expand All @@ -52,8 +40,8 @@ fmt: ## Format codes.

.PHONY: test
test: ## Build and run unit tests.
cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_CCACHE=ON
cmake --build build --parallel ${COMPILE_JOBS}
cmake ${VSAG_CMAKE_ARGS} -B${DEBUG_BUILD_DIR} -DCMAKE_BUILD_TYPE=Debug -DENABLE_CCACHE=ON
cmake --build ${DEBUG_BUILD_DIR} --parallel ${COMPILE_JOBS}
./build/tests/unittests -d yes ${UT_FILTER} --allow-running-no-tests ${UT_SHARD}
./build/tests/functests -d yes ${UT_FILTER} --allow-running-no-tests ${UT_SHARD}
./build/mockimpl/tests_mockimpl -d yes ${UT_FILTER} --allow-running-no-tests ${UT_SHARD}
Expand All @@ -65,8 +53,8 @@ test_parallel: debug

.PHONY: asan
asan: ## Build with AddressSanitizer option.
cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=ON -DENABLE_CCACHE=ON
cmake --build build --parallel ${COMPILE_JOBS}
cmake ${VSAG_CMAKE_ARGS} -B${DEBUG_BUILD_DIR} -DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=ON -DENABLE_CCACHE=ON
cmake --build ${DEBUG_BUILD_DIR} --parallel ${COMPILE_JOBS}

.PHONY: test_asan_parallel
test_asan_parallel: asan ## Run unit tests parallel with AddressSanitizer option.
Expand All @@ -81,8 +69,8 @@ test_asan: asan ## Run unit tests with AddressSanitizer option.

.PHONY: tsan
tsan: ## Build with ThreadSanitizer option.
cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_TSAN=ON -DENABLE_CCACHE=ON
cmake --build build --parallel ${COMPILE_JOBS}
cmake ${VSAG_CMAKE_ARGS} -B${DEBUG_BUILD_DIR} -DCMAKE_BUILD_TYPE=Debug -DENABLE_TSAN=ON -DENABLE_CCACHE=ON
cmake --build ${DEBUG_BUILD_DIR} --parallel ${COMPILE_JOBS}

.PHONY: test_tsan
test_tsan: tsan ## Run unit tests with ThreadSanitizer option.
Expand All @@ -92,8 +80,8 @@ test_tsan: tsan ## Run unit tests with ThreadSanitizer option.

.PHONY: cov # Build unit tests with code coverage enabled.
cov:
cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=ON -DENABLE_CCACHE=ON
cmake --build build --parallel ${COMPILE_JOBS}
cmake ${VSAG_CMAKE_ARGS} -B${DEBUG_BUILD_DIR} -DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=ON -DENABLE_CCACHE=ON
cmake --build ${DEBUG_BUILD_DIR} --parallel ${COMPILE_JOBS}

.PHONY: test_cov
test_cov: cov ## Build and run unit tests with code coverage enabled.
Expand All @@ -105,8 +93,33 @@ test_cov: cov ## Build and run unit tests with code coverage enabled.

.PHONY: clean
clean: ## Clear build/ directory.
rm -rf build/*
rm -rf ${DEBUG_BUILD_DIR}/*

# ================= distribution part =================
.PHONY: release
release: ## Build vsag with release options.
cmake ${VSAG_CMAKE_ARGS} -B${RELEASE_BUILD_DIR} -DCMAKE_BUILD_TYPE=Release
cmake --build ${RELEASE_BUILD_DIR} --parallel ${COMPILE_JOBS}

.PHONY: distribution
distribution: ## Build vsag with distribution options.
cmake ${VSAG_CMAKE_ARGS} -B${RELEASE_BUILD_DIR} -DCMAKE_BUILD_TYPE=Release -DENABLE_CXX11_ABI=off -DENABLE_LIBCXX=off
cmake --build ${RELEASE_BUILD_DIR} --parallel ${COMPILE_JOBS}

.PHONY: libcxx
libcxx: ## Build vsag using libc++.
cmake ${VSAG_CMAKE_ARGS} -B${RELEASE_BUILD_DIR} -DCMAKE_BUILD_TYPE=Release -DENABLE_LIBCXX=on
cmake --build ${RELEASE_BUILD_DIR} --parallel ${COMPILE_JOBS}

.PHONY: install
install: ## Build and install the release version of vsag.
cmake --install build/
cmake --install ${RELEASE_BUILD_DIR}/


PARAM1 := "-DNUM_BUILDING_JOBS=${COMPILE_JOBS} -DENABLE_PYBINDS=1 -S. -B${RELEASE_BUILD_DIR} -DCMAKE_BUILD_TYPE=Release"
PARAM2 := "--build ${RELEASE_BUILD_DIR} --parallel ${COMPILE_JOBS}"
PARAM3 := "${RELEASE_BUILD_DIR}"

.PHONY: pyvsag ## Build pyvsag wheel
pyvsag:
bash ./scripts/build_pyvsag_multiple_version.sh $(PARAM1) $(PARAM2) $(PARAM3)
37 changes: 37 additions & 0 deletions python/example.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

// Copyright 2024-present the vsag project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <Python.h>

static PyObject* hello(PyObject* self, PyObject* args) {
return Py_BuildValue("s", "Hello, world!");
}

static PyMethodDef ExampleMethods[] = {
{"hello", hello, METH_VARARGS, "Greet the world."},
{NULL, NULL, 0, NULL}
};

static struct PyModuleDef examplemodule = {
PyModuleDef_HEAD_INIT,
"example",
NULL,
-1,
ExampleMethods
};

PyMODINIT_FUNC PyInit_example(void) {
return PyModule_Create(&examplemodule);
}
9 changes: 9 additions & 0 deletions python/pyvsag/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@

import os
import sys

_cur_file_dir = os.path.dirname(os.path.realpath(__file__))
sys.path.append(_cur_file_dir)

from _pyvsag import *
from ._version import __version__
2 changes: 2 additions & 0 deletions python/pyvsag/_version.py.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@

__version__ = "@PYVSAG_VERSION@"
2 changes: 2 additions & 0 deletions python/setup-requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
setuptools_scm
pep440
82 changes: 82 additions & 0 deletions python/setup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@

# Copyright 2024-present the vsag project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
from setuptools import setup, find_packages, Extension
import os
import shutil
import platform


long_description="""VSAG is a vector indexing library used for similarity search. The indexing algorithm allows users to search through various sizes of vector sets, especially those that cannot fit in memory. The library also provides methods for generating parameters based on vector dimensions and data scale, allowing developers to use it without understanding the algorithm’s principles. VSAG is written in C++ and provides a Python wrapper package called pyvsag. Developed by the Vector Database Team at Ant Group."""


# DON'T REMOVE: to make the wheel's name contains python version
example_module = Extension('example', sources=['example.c'])

def get_version():
from setuptools_scm import get_version as scm_get_version
from setuptools_scm.version import get_local_node_and_date, get_no_local_node
from pep440 import is_canonical

# the package with local version is not allowed to be uploaded to the
# pypi. set build_local_version=True if you're building a local wheel.
build_local_version = False
local_scheme = get_no_local_node
if build_local_version:
local_scheme = get_local_node_and_date

version = scm_get_version(root=f'{__file__}/../..', local_scheme=local_scheme)
version_file = os.path.join(os.path.dirname(__file__), 'pyvsag', '_version.py')
with open(version_file, 'w') as f:
f.write(f"\n__version__ = '{version}'\n")

# make sure the publish version is correct
if not build_local_version and not is_canonical(version):
print(f"!!\n\tversion {version} is incorrect, exit\n!!")
exit(1)

return version

setup(
name='pyvsag',
version=get_version(),
description='vsag is a vector indexing library used for similarity search',
long_description=long_description,
url='https://github.com/antgroup/vsag',
author='the vsag project',
author_email='[email protected]',
license='Apache-2.0',
keywords='search nearest neighbors',
install_requires=['packaging'],
packages=find_packages(),
package_data={
'': ['*.so', '*.so.*'],
},
include_package_data=True,
zip_safe=False,
ext_modules=[example_module],
classifiers=[
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
],
)
Loading

0 comments on commit b12a120

Please sign in to comment.