Implement order preserving transform for inner product search (#25) #195
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compile, Test, and Deploy | |
| on: | |
| pull_request: {} | |
| push: | |
| branches: | |
| - main | |
| release: | |
| types: [published] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint-java: | |
| runs-on: 'ubuntu-latest' | |
| continue-on-error: true | |
| name: Lint Java | |
| defaults: | |
| run: | |
| working-directory: java | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '11' | |
| distribution: 'corretto' | |
| - name: Check Formatting | |
| run: mvn --batch-mode com.spotify.fmt:fmt-maven-plugin:check | |
| lint-cpp: | |
| runs-on: 'ubuntu-latest' | |
| continue-on-error: true | |
| name: Lint C++ | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Check C++ Formatting | |
| uses: jidicula/[email protected] | |
| with: | |
| clang-format-version: 14 | |
| fallback-style: LLVM | |
| run-java-tests: | |
| continue-on-error: true | |
| name: Test with Java ${{ matrix.java-version }} on ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| working-directory: java | |
| strategy: | |
| matrix: | |
| os: ['ubuntu-latest', windows-latest, macos-12] | |
| java-version: ['11', '17'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: 'corretto' | |
| gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| - uses: ilammy/msvc-dev-cmd@v1 | |
| if: runner.os == 'Windows' | |
| - name: Compile | |
| run: make | |
| - name: Run Tests | |
| run: mvn --batch-mode verify | |
| env: | |
| MAVEN_GPG_PASSPHRASE: ${{ secrets.SONATYPE_PASSPHRASE }} | |
| run-python-tests: | |
| runs-on: ${{ matrix.os }} | |
| continue-on-error: true | |
| defaults: | |
| run: | |
| working-directory: python | |
| strategy: | |
| matrix: | |
| python-version: | |
| # - '3.7' | |
| - '3.8' | |
| # - '3.9' | |
| # - '3.10' | |
| # - '3.11' | |
| # TODO: Switch back to macos-latest once https://github.com/actions/python-versions/pull/114 is fixed | |
| os: ['ubuntu-latest', windows-latest, macos-12] | |
| name: Test with Python ${{ matrix.python-version }} on ${{ matrix.os }} | |
| steps: | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Install Linux dependencies | |
| if: runner.os == 'Linux' | |
| run: sudo apt-get update && sudo apt-get install -y pkg-config | |
| - name: Install test dependencies | |
| env: | |
| # on macOS and with Python 3.10: building NumPy from source fails without these options: | |
| NPY_BLAS_ORDER: "" | |
| NPY_LAPACK_ORDER: "" | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install wheel | |
| pip install -r dev-requirements.txt | |
| - name: Build voyager locally | |
| run: python setup.py develop | |
| - name: Run tests | |
| run: pytest -v --durations=100 -x | |
| run-python-tests-with-address-sanitizer: | |
| defaults: | |
| run: | |
| working-directory: python | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| python-version: | |
| # - '3.7' | |
| - '3.8' | |
| # - '3.9' | |
| # - '3.10' | |
| # - '3.11' | |
| os: ['ubuntu-latest'] | |
| name: Test with Python ${{ matrix.python-version }} + Address Sanitizer | |
| steps: | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Install Linux dependencies | |
| if: runner.os == 'Linux' | |
| run: sudo apt-get update && sudo apt-get install -y pkg-config | |
| - name: Install test dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install wheel | |
| pip install -r dev-requirements.txt | |
| - name: Build voyager locally | |
| env: | |
| DEBUG: "0" | |
| USE_ASAN: "1" | |
| CC: clang | |
| CXX: clang++ | |
| run: python setup.py develop | |
| - name: Run tests with ASan loaded | |
| # pytest can exit before all Python objects have been destroyed, | |
| # so we tell ASan to ignore leaks. | |
| run: | | |
| ASAN_OPTIONS=detect_leaks=0 \ | |
| LD_PRELOAD=$(clang -print-file-name=libclang_rt.asan-x86_64.so) \ | |
| pytest -v -x -s | |
| build-python-wheels: | |
| needs: [run-python-tests, run-python-tests-with-address-sanitizer] | |
| runs-on: ${{ matrix.os }} | |
| continue-on-error: true | |
| if: true # (github.event_name == 'release' && github.event.action == 'published') || contains(github.event.pull_request.labels.*.name, 'Also Test Wheels') | |
| strategy: | |
| matrix: | |
| include: | |
| - { os: macos-12, build: cp37-macosx_x86_64 } | |
| - { os: macos-12, build: cp38-macosx_x86_64 } | |
| - { os: macos-12, build: cp39-macosx_x86_64 } | |
| - { os: macos-12, build: cp310-macosx_x86_64 } | |
| - { os: macos-12, build: cp311-macosx_x86_64 } | |
| - { os: macos-12, build: cp312-macosx_x86_64 } | |
| - { os: macos-12, build: cp38-macosx_universal2 } | |
| - { os: macos-12, build: cp39-macosx_universal2 } | |
| - { os: macos-12, build: cp310-macosx_universal2 } | |
| - { os: macos-12, build: cp311-macosx_universal2 } | |
| - { os: macos-12, build: cp312-macosx_universal2 } | |
| - { os: macos-12, build: cp38-macosx_arm64 } | |
| - { os: macos-12, build: cp39-macosx_arm64 } | |
| - { os: macos-12, build: cp310-macosx_arm64 } | |
| - { os: macos-12, build: cp311-macosx_arm64 } | |
| - { os: macos-12, build: cp312-macosx_arm64 } | |
| # - { os: macos-12, build: pp37-macosx_x86_64 } | |
| # - { os: macos-12, build: pp38-macosx_x86_64 } | |
| # - { os: macos-12, build: pp39-macosx_x86_64 } | |
| - { os: windows-latest, build: cp37-win_amd64 } | |
| - { os: windows-latest, build: cp38-win_amd64 } | |
| - { os: windows-latest, build: cp39-win_amd64 } | |
| - { os: windows-latest, build: cp310-win_amd64 } | |
| - { os: windows-latest, build: cp311-win_amd64 } | |
| - { os: windows-latest, build: cp312-win_amd64 } | |
| # - { os: windows-latest, build: pp37-win_amd64 } | |
| # - { os: windows-latest, build: pp38-win_amd64 } | |
| # - { os: windows-latest, build: pp39-win_amd64 } | |
| - { os: 'ubuntu-latest', build: cp37-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp37-manylinux_aarch64 } | |
| - { os: 'ubuntu-latest', build: cp38-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp38-manylinux_aarch64 } | |
| - { os: 'ubuntu-latest', build: cp39-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp39-manylinux_aarch64 } | |
| - { os: 'ubuntu-latest', build: cp310-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp310-manylinux_aarch64 } | |
| - { os: 'ubuntu-latest', build: cp311-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp311-manylinux_aarch64 } | |
| - { os: 'ubuntu-latest', build: cp312-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp312-manylinux_aarch64 } | |
| # - { os: 'ubuntu-latest', build: pp37-manylinux_x86_64 } | |
| # - { os: 'ubuntu-latest', build: pp37-manylinux_aarch64 } | |
| # - { os: 'ubuntu-latest', build: pp38-manylinux_x86_64 } | |
| # - { os: 'ubuntu-latest', build: pp38-manylinux_aarch64 } | |
| # - { os: 'ubuntu-latest', build: pp39-manylinux_x86_64 } | |
| # - { os: 'ubuntu-latest', build: pp39-manylinux_aarch64 } | |
| name: Build wheel for ${{ matrix.build }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # Used to host cibuildwheel, so version doesn't really matter | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| - name: Install cibuildwheel | |
| run: python -m pip install 'cibuildwheel>=2.11.0' | |
| - name: Copy README into place | |
| run: cp README.md python/ | |
| - name: Set up QEMU for aarch64 on Linux | |
| if: runner.os == 'Linux' | |
| uses: docker/setup-qemu-action@v1 | |
| with: | |
| platforms: all | |
| - name: Build wheels | |
| run: cp -r cpp python/cpp && cd python && python -m cibuildwheel --output-dir ../wheelhouse | |
| env: | |
| CIBW_TEST_REQUIRES: -r dev-requirements.txt | |
| CIBW_TEST_COMMAND: "pytest {project}/tests -x" | |
| CIBW_BUILD: ${{ matrix.build }} | |
| CIBW_ARCHS: auto64 # Only support building 64-bit wheels. It's 2022! | |
| CIBW_ARCHS_LINUX: auto64 aarch64 # Useful for building linux images with Apple Silicon | |
| CIBW_ARCHS_MACOS: x86_64 universal2 arm64 # Support Apple Silicon | |
| CIBW_ARCHS_WINDOWS: AMD64 | |
| # on macOS and with Python 3.10: building NumPy from source fails without these options: | |
| CIBW_ENVIRONMENT: NPY_BLAS_ORDER="" NPY_LAPACK_ORDER="" | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: pip install auditwheel-symbols && (auditwheel repair -w {dest_dir} {wheel} || auditwheel-symbols --manylinux 2010 {wheel}) | |
| # The manylinux container doesn't have a new enough glibc version, | |
| # so we can't run post-wheel-build tests there. | |
| # Also testing any pypy versions fails, as TensorFlow isn't pypy compatible. | |
| CIBW_TEST_SKIP: "*manylinux* *pp* *-macosx_universal2:arm64" | |
| # Use the minimum macOS deployment target that has C++17 support: | |
| MACOSX_DEPLOYMENT_TARGET: "10.13" | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| if-no-files-found: error | |
| name: python-wheels | |
| path: ./wheelhouse/*.whl | |
| build-java-jars: | |
| continue-on-error: false | |
| name: Build JAR with on ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| working-directory: java | |
| strategy: | |
| matrix: | |
| os: ['ubuntu-latest', windows-latest, macos-12] | |
| java-version: ['11'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: 'corretto' | |
| gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| - name: Set up QEMU for aarch64 on Linux | |
| if: runner.os == 'Linux' | |
| uses: docker/setup-qemu-action@v1 | |
| with: | |
| platforms: all | |
| - uses: ilammy/msvc-dev-cmd@v1 | |
| if: runner.os == 'Windows' | |
| - name: Compile | |
| run: make | |
| - name: Package | |
| run: mvn --batch-mode package | |
| env: | |
| MAVEN_GPG_PASSPHRASE: ${{ secrets.SONATYPE_PASSPHRASE }} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| if-no-files-found: error | |
| name: java-${{ matrix.os }} | |
| path: java/target/voyager-*.jar | |
| combine-java-jars: | |
| name: Combine JARs | |
| runs-on: ubuntu-latest | |
| needs: build-java-jars | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| path: artifacts | |
| - name: Unpack and Repack JARs | |
| shell: bash | |
| working-directory: artifacts | |
| run: | | |
| rm -fv java-*/*-sources.jar | |
| rm -fv java-*/*-javadoc.jar | |
| for artifact in java-*; do unzip -o $artifact/*.jar -d combined-jar; done | |
| # Quick test that the JARs were built for the correct architectures | |
| echo "Checking linux-x64/libvoyager.so for x86 architecture..." | |
| file combined-jar/linux-x64/libvoyager.so | grep x86 | |
| echo "Checking mac-x64/libvoyager.dylib for x86 architecture..." | |
| file combined-jar/mac-x64/libvoyager.dylib | grep x86 | |
| echo "Checking linux-aarch64/libvoyager.so for aarch64 architecture..." | |
| file combined-jar/linux-aarch64/libvoyager.so | grep aarch64 | |
| echo "Checking mac-aarch64/libvoyager.dylib for arm64 architecture..." | |
| file combined-jar/mac-aarch64/libvoyager.dylib | grep arm64 | |
| # Check the Linux JARs to ensure a sufficiently old required GLIBC and GLIBCXX version: | |
| echo "Checking linux-aarch64/libvoyager.so for GLIBC 2.27..." | |
| objdump -T combined-jar/linux-aarch64/libvoyager.so | grep GLIBC | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -n 1 | grep 2\.27 | |
| echo "Checking linux-aarch64/libvoyager.so for GLIBCXX 3.4.22..." | |
| objdump -T combined-jar/linux-aarch64/libvoyager.so | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/\1/g' | sort -Vu | tail -n 1 | grep 3\.4\.22 | |
| echo "Checking linux-x64/libvoyager.so for GLIBC 2.29..." | |
| objdump -T combined-jar/linux-x64/libvoyager.so | grep GLIBC | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -n 1 | grep 2\.29 | |
| echo "Checking linux-x64/libvoyager.so for GLIBCXX 3.4.22..." | |
| objdump -T combined-jar/linux-x64/libvoyager.so | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/\1/g' | sort -Vu | tail -n 1 | grep 3\.4\.22 | |
| zip -r $(ls java-* | grep jar | tail -n 1) combined-jar/* | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| if-no-files-found: error | |
| name: java-all-platforms | |
| path: artifacts/*.jar | |
| upload-pypi: | |
| defaults: | |
| run: | |
| working-directory: python | |
| needs: [build-python-wheels, run-python-tests-with-address-sanitizer, run-python-tests] | |
| runs-on: 'ubuntu-latest' | |
| name: "Upload wheels to PyPI" | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: python-wheels | |
| path: dist | |
| - uses: pypa/[email protected] | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_DEPLOY_TOKEN }} | |
| upload-maven: | |
| name: Upload to Maven Central | |
| runs-on: ubuntu-latest | |
| needs: combine-java-jars | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| defaults: | |
| run: | |
| working-directory: java | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: java-all-platforms | |
| path: combined-jar | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: 11 | |
| distribution: 'corretto' | |
| server-id: ossrh | |
| server-username: MAVEN_USERNAME | |
| server-password: MAVEN_PASSWORD | |
| gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| - name: Deploy | |
| # Note: this will re-run the build even though we already have the JAR file ready to deploy | |
| run: | | |
| # Copy the native artifacts built for each platform into the appropriate place: | |
| unzip ../combined-jar/*.jar -d ../ | |
| mkdir -p target/classes/ | |
| cp -r ../combined-jar/linux* target/classes/ | |
| cp -r ../combined-jar/mac* target/classes/ | |
| cp -r ../combined-jar/win* target/classes/ | |
| mvn --batch-mode deploy -Dmaven.test.skip=true | |
| env: | |
| MAVEN_USERNAME: ${{ secrets.SONATYPE_USERNAME }} | |
| MAVEN_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} | |
| MAVEN_GPG_PASSPHRASE: ${{ secrets.SONATYPE_PASSPHRASE }} |