Skip to content

C++ improvements

C++ improvements #441

Workflow file for this run

name: Compile, Test, and Deploy
on:
pull_request: {}
push:
branches:
- main
release:
types: [published]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
lint-java:
runs-on: 'ubuntu-latest'
continue-on-error: true
name: Lint Java
defaults:
run:
working-directory: java
steps:
- uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'corretto'
- name: Check Formatting
run: mvn --batch-mode com.spotify.fmt:fmt-maven-plugin:check
lint-cpp:
runs-on: 'ubuntu-latest'
continue-on-error: true
name: Lint C++
steps:
- uses: actions/checkout@v3
- name: Check C++ Formatting
uses: jidicula/[email protected]
with:
clang-format-version: 16
run-java-tests:
continue-on-error: true
name: Test with Java ${{ matrix.java-version }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
defaults:
run:
working-directory: java
strategy:
matrix:
os: ['ubuntu-latest', windows-latest, macos-12]
java-version: ['11', '17']
steps:
- uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.java-version }}
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.java-version }}
distribution: 'corretto'
gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
- uses: ilammy/msvc-dev-cmd@v1
if: runner.os == 'Windows'
- name: Compile
run: make
- name: Run Tests
run: mvn --batch-mode test
run-java-tests-with-address-sanitizer:
if: false # Disabled due to ASan log spam in output
continue-on-error: true
name: Test with Java ${{ matrix.java-version }} + Address Sanitizer
runs-on: ubuntu-latest
defaults:
run:
working-directory: java
strategy:
matrix:
java-version: ['11', '17']
steps:
- uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.java-version }}
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'corretto'
gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
- name: Compile
env:
USE_ASAN: "1"
run: make target/classes/linux-x64/libvoyager.so
- name: Run Tests
run: |
ASAN_OPTIONS=detect_leaks=0
LD_PRELOAD=$(clang -print-file-name=libclang_rt.asan-x86_64.so) \
mvn --batch-mode test
run-python-tests:
runs-on: ${{ matrix.os }}
continue-on-error: true
defaults:
run:
working-directory: python
strategy:
matrix:
python-version:
# - '3.7'
- '3.8'
# - '3.9'
# - '3.10'
# - '3.11'
# TODO: Switch back to macos-latest once https://github.com/actions/python-versions/pull/114 is fixed
os: ['ubuntu-latest', windows-latest, macos-12]
name: Test with Python ${{ matrix.python-version }} on ${{ matrix.os }}
steps:
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Install Linux dependencies
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y pkg-config
- name: Install test dependencies
env:
# on macOS and with Python 3.10: building NumPy from source fails without these options:
NPY_BLAS_ORDER: ""
NPY_LAPACK_ORDER: ""
run: |
python -m pip install --upgrade pip
pip install wheel
pip install -r dev-requirements.txt
- name: Build voyager locally
run: python setup.py develop
- name: Run tox tests
run: tox
run-python-tests-with-address-sanitizer:
defaults:
run:
working-directory: python
runs-on: ${{ matrix.os }}
strategy:
matrix:
python-version:
# - '3.7'
- '3.8'
# - '3.9'
# - '3.10'
# - '3.11'
os: ['ubuntu-latest']
name: Test with Python ${{ matrix.python-version }} + Address Sanitizer
steps:
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Install Linux dependencies
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y pkg-config
- name: Install test dependencies
run: |
python -m pip install --upgrade pip
pip install wheel
pip install -r dev-requirements.txt
- name: Build voyager locally
env:
DEBUG: "0"
USE_ASAN: "1"
CC: clang
CXX: clang++
run: python setup.py develop
- name: Run tox tests with ASan loaded
# pytest can exit before all Python objects have been destroyed,
# so we tell ASan to ignore leaks.
run: |
ASAN_OPTIONS=detect_leaks=0 \
LD_PRELOAD=$(clang -print-file-name=libclang_rt.asan-x86_64.so) \
tox
run-benchmarks:
strategy:
matrix:
os: ['ubuntu-latest']
runs-on: ${{ matrix.os }}
continue-on-error: true
if: github.event_name == 'pull_request'
name: Run benchmarks on ${{ matrix.os }}
steps:
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: '3.9'
- uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: recursive
- name: Install Linux dependencies
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y pkg-config
- name: Install dev dependencies
run: |
python -m pip install --upgrade pip
pip install wheel
pip install -r python/dev-requirements.txt
- name: Run benchmarks
run: |
asv machine --yes
asv continuous --sort name --no-only-changed refs/remotes/origin/main ${{ github.sha }} | tee >(sed '1,/All benchmarks:/d' > $GITHUB_STEP_SUMMARY)
run-cpp-tests:
runs-on: ${{ matrix.os }}
continue-on-error: true
defaults:
run:
working-directory: cpp
strategy:
matrix:
# TODO: Switch back to macos-latest once https://github.com/actions/python-versions/pull/114 is fixed
os:
- 'ubuntu-latest'
# TODO: Fix failing CMake build on windows:
# `cl : command line error D8016: '/O2' and '/RTC1' command-line options are incompatible [D:\a\voyager\voyager\cpp\test\test.vcxproj]`
- windows-latest
- macos-12
name: Test C++ on ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Set up CMake on Windows
if: matrix.os == 'windows-latest'
run: |
choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System'
choco install ninja
- name: Set up CMake on macOS
if: matrix.os == 'macos-12'
run: brew install cmake
- name: Set up CMake on Ubuntu
if: matrix.os == 'ubuntu-latest'
run: sudo apt-get install -y cmake
- name: Configure CMake
run: cmake -S . -B build
- name: Build with CMake
run: cmake --build build
- name: Run Tests (if any)
run: ctest --test-dir build
# - name: Run CMake (Linux & MacOS)
# if: runner.os != 'Windows'
# uses: threeal/[email protected]
# with:
# source-dir: cpp
# build-dir: cpp
# - name: Run CMake (Windows)
# if: runner.os == 'Windows'
# uses: threeal/[email protected]
# with:
# source-dir: cpp
# build-dir: cpp
# cxx-flags: /Od # Disable optimization (since it breaks CMake on Windows)
# - name: Build C++
# run: make test
# - name: Run tests
# run: ./test/test
build-python-wheels:
needs: [run-python-tests, run-python-tests-with-address-sanitizer]
runs-on: ${{ matrix.os }}
continue-on-error: true
if: true # (github.event_name == 'release' && github.event.action == 'published') || contains(github.event.pull_request.labels.*.name, 'Also Test Wheels')
strategy:
matrix:
include:
- { os: macos-12, build: cp37-macosx_x86_64 }
- { os: macos-12, build: cp38-macosx_x86_64 }
- { os: macos-12, build: cp39-macosx_x86_64 }
- { os: macos-12, build: cp310-macosx_x86_64 }
- { os: macos-12, build: cp311-macosx_x86_64 }
- { os: macos-12, build: cp312-macosx_x86_64 }
- { os: macos-12, build: cp38-macosx_universal2 }
- { os: macos-12, build: cp39-macosx_universal2 }
- { os: macos-12, build: cp310-macosx_universal2 }
- { os: macos-12, build: cp311-macosx_universal2 }
- { os: macos-12, build: cp312-macosx_universal2 }
- { os: macos-12, build: cp38-macosx_arm64 }
- { os: macos-12, build: cp39-macosx_arm64 }
- { os: macos-12, build: cp310-macosx_arm64 }
- { os: macos-12, build: cp311-macosx_arm64 }
- { os: macos-12, build: cp312-macosx_arm64 }
# - { os: macos-12, build: pp37-macosx_x86_64 }
# - { os: macos-12, build: pp38-macosx_x86_64 }
# - { os: macos-12, build: pp39-macosx_x86_64 }
- { os: windows-latest, build: cp37-win_amd64 }
- { os: windows-latest, build: cp38-win_amd64 }
- { os: windows-latest, build: cp39-win_amd64 }
- { os: windows-latest, build: cp310-win_amd64 }
- { os: windows-latest, build: cp311-win_amd64 }
- { os: windows-latest, build: cp312-win_amd64 }
# - { os: windows-latest, build: pp37-win_amd64 }
# - { os: windows-latest, build: pp38-win_amd64 }
# - { os: windows-latest, build: pp39-win_amd64 }
- { os: 'ubuntu-latest', build: cp37-manylinux_x86_64 }
- { os: 'ubuntu-latest', build: cp37-manylinux_aarch64 }
- { os: 'ubuntu-latest', build: cp38-manylinux_x86_64 }
- { os: 'ubuntu-latest', build: cp38-manylinux_aarch64 }
- { os: 'ubuntu-latest', build: cp39-manylinux_x86_64 }
- { os: 'ubuntu-latest', build: cp39-manylinux_aarch64 }
- { os: 'ubuntu-latest', build: cp310-manylinux_x86_64 }
- { os: 'ubuntu-latest', build: cp310-manylinux_aarch64 }
- { os: 'ubuntu-latest', build: cp311-manylinux_x86_64 }
- { os: 'ubuntu-latest', build: cp311-manylinux_aarch64 }
- { os: 'ubuntu-latest', build: cp312-manylinux_x86_64 }
- { os: 'ubuntu-latest', build: cp312-manylinux_aarch64 }
# - { os: 'ubuntu-latest', build: pp37-manylinux_x86_64 }
# - { os: 'ubuntu-latest', build: pp37-manylinux_aarch64 }
# - { os: 'ubuntu-latest', build: pp38-manylinux_x86_64 }
# - { os: 'ubuntu-latest', build: pp38-manylinux_aarch64 }
# - { os: 'ubuntu-latest', build: pp39-manylinux_x86_64 }
# - { os: 'ubuntu-latest', build: pp39-manylinux_aarch64 }
name: Build wheel for ${{ matrix.build }}
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
# Used to host cibuildwheel, so version doesn't really matter
- uses: actions/setup-python@v3
with:
python-version: "3.10"
- name: Install cibuildwheel
run: python -m pip install 'cibuildwheel>=2.11.0'
- name: Copy README into place
run: cp README.md python/
- name: Set up QEMU for aarch64 on Linux
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v1
with:
platforms: all
- name: Build wheels
run: cp -r cpp python/cpp && cd python && python -m cibuildwheel --output-dir ../wheelhouse
env:
CIBW_TEST_REQUIRES: -r dev-requirements.txt
CIBW_TEST_COMMAND: "pytest {project}/tests -x"
CIBW_BUILD: ${{ matrix.build }}
CIBW_ARCHS: auto64 # Only support building 64-bit wheels. It's 2022!
CIBW_ARCHS_LINUX: auto64 aarch64 # Useful for building linux images with Apple Silicon
CIBW_ARCHS_MACOS: x86_64 universal2 arm64 # Support Apple Silicon
CIBW_ARCHS_WINDOWS: AMD64
# on macOS and with Python 3.10: building NumPy from source fails without these options:
CIBW_ENVIRONMENT: NPY_BLAS_ORDER="" NPY_LAPACK_ORDER=""
CIBW_REPAIR_WHEEL_COMMAND_LINUX: pip install auditwheel-symbols && (auditwheel repair -w {dest_dir} {wheel} || auditwheel-symbols --manylinux 2010 {wheel})
# Use the minimum macOS deployment target that has C++17 support:
CIBW_TEST_SKIP: "*aarch64"
MACOSX_DEPLOYMENT_TARGET: "10.13"
- uses: actions/upload-artifact@v3
with:
if-no-files-found: error
name: python-wheels
path: ./wheelhouse/*.whl
build-java-jars:
continue-on-error: false
name: Build JAR with on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
defaults:
run:
working-directory: java
strategy:
matrix:
os: ['ubuntu-latest', windows-latest, macos-12]
java-version: ['11']
steps:
- uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.java-version }}
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.java-version }}
distribution: 'corretto'
gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
- name: Set up QEMU for aarch64 on Linux
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v1
with:
platforms: all
- uses: ilammy/msvc-dev-cmd@v1
if: runner.os == 'Windows'
- name: Compile
run: make
- name: Package
run: mvn --batch-mode package
env:
MAVEN_GPG_PASSPHRASE: ${{ secrets.SONATYPE_PASSPHRASE }}
- uses: actions/upload-artifact@v3
with:
if-no-files-found: error
name: java-${{ matrix.os }}
path: java/target/voyager-*.jar
combine-java-jars:
name: Combine JARs
runs-on: ubuntu-latest
needs: build-java-jars
steps:
- uses: actions/download-artifact@v3
with:
path: artifacts
- name: Unpack and Repack JARs
shell: bash
working-directory: artifacts
run: |
rm -fv java-*/*-sources.jar
rm -fv java-*/*-javadoc.jar
for artifact in java-*; do unzip -o $artifact/*.jar -d combined-jar; done
# Quick test that the JARs were built for the correct architectures
echo "Checking linux-x64/libvoyager.so for x86 architecture..."
file combined-jar/linux-x64/libvoyager.so | grep x86
echo "Checking mac-x64/libvoyager.dylib for x86 architecture..."
file combined-jar/mac-x64/libvoyager.dylib | grep x86
echo "Checking linux-aarch64/libvoyager.so for aarch64 architecture..."
file combined-jar/linux-aarch64/libvoyager.so | grep aarch64
echo "Checking mac-aarch64/libvoyager.dylib for arm64 architecture..."
file combined-jar/mac-aarch64/libvoyager.dylib | grep arm64
echo "Checking the Linux JARs to ensure a sufficiently old required GLIBC and GLIBCXX version in each..."
echo "Checking linux-aarch64/libvoyager.so (should require at most GLIBC 2.27...)"
objdump -T combined-jar/linux-aarch64/libvoyager.so | grep GLIBC | grep -v GLIBCXX | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | cat - <(echo 2\.27) | sort -Vu | tail -n 1 | grep 2\.27 || (echo "Expected linux-aarch64 binary to require at most glibc 2.27. Ensure the dockcross container has not been updated." && false)
echo "Checking linux-aarch64/libvoyager.so (should require at most GLIBCXX 3.4.22...)"
objdump -T combined-jar/linux-aarch64/libvoyager.so | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/\1/g' | cat - <(echo 3\.4\.22) | sort -Vu | tail -n 1 | grep 3\.4\.22 || (echo "Expected linux-aarch64 binary to require at most glibc++ 3.4.22. Ensure the dockcross container has not been updated." && false)
echo "Checking linux-x64/libvoyager.so (should require at most GLIBC 2.27...)"
objdump -T combined-jar/linux-x64/libvoyager.so | grep GLIBC | grep -v GLIBCXX | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | cat - <(echo 2\.27) | sort -Vu | tail -n 1 | grep 2\.27 || (echo "Expected linux-x64 binary to require at most glibc 2.28. Ensure the dockcross container has not been updated." && false)
echo "Checking linux-x64/libvoyager.so (should require at most GLIBCXX 3.4.22...)"
objdump -T combined-jar/linux-x64/libvoyager.so | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/\1/g' | cat - <(echo 3\.4\.22) | sort -Vu | tail -n 1 | grep 3\.4\.22 || (echo "Expected linux-x64 binary to require at most glibc++ 3.4.22. Ensure the dockcross container has not been updated." && false)
zip -r $(ls java-* | grep jar | tail -n 1) combined-jar/*
- uses: actions/upload-artifact@v3
with:
if-no-files-found: error
name: java-all-platforms
path: artifacts/*.jar
upload-pypi:
defaults:
run:
working-directory: python
needs: [build-python-wheels, run-python-tests-with-address-sanitizer, run-python-tests]
runs-on: 'ubuntu-latest'
name: "Upload wheels to PyPI"
if: github.event_name == 'release' && github.event.action == 'published'
steps:
- uses: actions/download-artifact@v3
with:
name: python-wheels
path: dist
- uses: pypa/[email protected]
with:
user: __token__
password: ${{ secrets.PYPI_DEPLOY_TOKEN }}
upload-maven:
name: Upload to Maven Central
runs-on: ubuntu-latest
needs: combine-java-jars
if: github.event_name == 'release' && github.event.action == 'published'
defaults:
run:
working-directory: java
steps:
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: java-all-platforms
path: combined-jar
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
java-version: 11
distribution: 'corretto'
server-id: ossrh
server-username: MAVEN_USERNAME
server-password: MAVEN_USER_TOKEN
gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
- name: Deploy
# Note: this will re-run the build even though we already have the JAR file ready to deploy
run: |
# Copy the native artifacts built for each platform into the appropriate place:
unzip ../combined-jar/*.jar -d ../
mkdir -p target/classes/
cp -r ../combined-jar/linux* target/classes/
cp -r ../combined-jar/mac* target/classes/
cp -r ../combined-jar/win* target/classes/
mvn --batch-mode deploy -Dmaven.test.skip=true
env:
MAVEN_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
MAVEN_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.SONATYPE_PASSPHRASE }}
MAVEN_USER_TOKEN: ${{ secrets.SONATYPE_USER_TOKEN }}