From 26a38ce96c9c057d18e79c4632c85d7272ebed26 Mon Sep 17 00:00:00 2001 From: gerrymanoim Date: Mon, 6 Jul 2020 10:23:21 -0400 Subject: [PATCH 01/15] BLD: Start testing on ubuntu 20.04 LTS --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3f26341a..88759578 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-18.04, macos-10.15] + os: [ubuntu-20.04, ubuntu-18.04, macos-10.15] python-version: [3.5, 3.6, 3.8] compiler: [gcc, clang] exclude: From 6016bf19843be6a8d12d04a61bfd6c3928aea07f Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Mon, 6 Jul 2020 10:46:34 -0400 Subject: [PATCH 02/15] BUG: PyDict_GetItemString returns a borrowed_ref --- tests/test_to_object.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_to_object.cc b/tests/test_to_object.cc index 388a7972..ccf7ce30 100644 --- a/tests/test_to_object.cc +++ b/tests/test_to_object.cc @@ -145,7 +145,7 @@ TEST_F(to_object, filesystem_path) { )"); ASSERT_TRUE(ns); - py::owned_ref py_path_ob{PyDict_GetItemString(ns.get(), "py_path")}; + py::borrowed_ref py_path_ob{PyDict_GetItemString(ns.get(), "py_path")}; ASSERT_TRUE(py_path_ob); #else py::owned_ref py_path_ob = py::to_object("/tmp/"); From d3eaba55803492dc8a01bc19f971b2dd1e0e0b07 Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Mon, 6 Jul 2020 11:15:10 -0400 Subject: [PATCH 03/15] Different apt depending on the ubuntu release --- .github/workflows/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 88759578..9968f89b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,11 +32,15 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'recursive' + - name: Set release name env variable (ubuntu) + if: startsWith(matrix.os, 'ubuntu') + run: | + echo ::set-env name=UBUNTU_RELEASE::$(lsb_release -sc) - name: Install newer clang (ubuntu) if: startsWith(matrix.os, 'ubuntu') && matrix.compiler == 'clang' run: | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key 2>/dev/null | sudo apt-key add - - sudo add-apt-repository 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' -y + sudo add-apt-repository 'deb http://apt.llvm.org/$UBUNTU_RELEASE/ llvm-toolchain-$UBUNTU_RELEASE-10 main' -y sudo apt-get update -q sudo apt-get install -y clang-10 lld-10 libc++-10-dev libc++abi-10-dev clang-tools-10 echo ::set-env name=AR::llvm-ar-10 From e333da67aee59b3b75958e9e8d73661f069df925 Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Mon, 6 Jul 2020 11:19:07 -0400 Subject: [PATCH 04/15] expand env vars --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9968f89b..1e4da0eb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,7 @@ jobs: if: startsWith(matrix.os, 'ubuntu') && matrix.compiler == 'clang' run: | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key 2>/dev/null | sudo apt-key add - - sudo add-apt-repository 'deb http://apt.llvm.org/$UBUNTU_RELEASE/ llvm-toolchain-$UBUNTU_RELEASE-10 main' -y + sudo add-apt-repository "deb http://apt.llvm.org/$UBUNTU_RELEASE/ llvm-toolchain-$UBUNTU_RELEASE-10 main" -y sudo apt-get update -q sudo apt-get install -y clang-10 lld-10 libc++-10-dev libc++abi-10-dev clang-tools-10 echo ::set-env name=AR::llvm-ar-10 From 5b0bbaa43a596b262d3c262e9372e33b706133bc Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Mon, 6 Jul 2020 12:07:16 -0400 Subject: [PATCH 05/15] use ld_flags.py instead of python-configure --- Makefile | 8 +++----- etc/ld_flags.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 etc/ld_flags.py diff --git a/Makefile b/Makefile index 839b3bb8..09e6e39e 100644 --- a/Makefile +++ b/Makefile @@ -62,10 +62,8 @@ CXXFLAGS = $(BASE_CXXFLAGS) $($(COMPILER)_FLAGS) # https://github.com/quantopian/libpy/pull/86/files#r309288697 INCLUDE_DIRS := include/ \ - $(shell $(PYTHON) -c "from distutils import sysconfig; \ - print(sysconfig.get_config_var('INCLUDEPY'))") \ - $(shell $(PYTHON) -c "from distutils import sysconfig; \ - print(sysconfig.get_config_var('INCLUDEDIR'))") \ + $(shell $(PYTHON) -c "import sysconfig; \ + print(sysconfig.get_config_var('INCLDIRSTOMAKE'))") \ $(shell $(PYTHON) -c 'import numpy as np;print(np.get_include())') \ $(EXTRA_INCLUDE_DIRS) INCLUDE := $(foreach d,$(INCLUDE_DIRS), -I$d) @@ -92,7 +90,7 @@ else CXXFLAGS += -fstack-protector-strong SONAME_FLAG := soname SONAME_PATH := $(SONAME) - LDFLAGS += $(shell $(PYTHON)-config --ldflags) + LDFLAGS += $(shell $(PYTHON) etc/ld_flags.py) LDFLAGS += -lstdc++fs LD_PRELOAD_VAR := LD_PRELOAD endif diff --git a/etc/ld_flags.py b/etc/ld_flags.py new file mode 100644 index 00000000..d2687b28 --- /dev/null +++ b/etc/ld_flags.py @@ -0,0 +1,14 @@ +# via https://github.com/python/cpython/blob/deb016224cc506503fb05e821a60158c83918ed4/Misc/python-config.in#L50 # noqa + +import sysconfig + +libs = [] +libpl = sysconfig.get_config_vars('LIBPL') +if libpl: + libs.append("-L"+libpl[0]) + +libpython = sysconfig.get_config_var('LIBPYTHON') +if libpython: + libs.append(libpython) +libs.extend(sysconfig.get_config_vars("LIBS", "SYSLIBS")) +print(' '.join(libs)) From 8bd42458140bf2c397125368038dd58bd83543b0 Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Mon, 6 Jul 2020 13:56:12 -0400 Subject: [PATCH 06/15] remove pre 3.4 compat, change construction --- tests/test_to_object.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_to_object.cc b/tests/test_to_object.cc index ccf7ce30..4257b991 100644 --- a/tests/test_to_object.cc +++ b/tests/test_to_object.cc @@ -138,19 +138,16 @@ TEST_F(to_object, filesystem_path) { std::filesystem::path test_path = "/tmp/"; py::owned_ref ob = py::to_object(test_path); ASSERT_TRUE(ob); -#if PY_VERSION_HEX >= 0x03040000 + py::owned_ref ns = RUN_PYTHON(R"( from pathlib import Path py_path = Path("/tmp/") )"); ASSERT_TRUE(ns); - py::borrowed_ref py_path_ob{PyDict_GetItemString(ns.get(), "py_path")}; + py::borrowed_ref py_path_ob = PyDict_GetItemString(ns.get(), "py_path"); ASSERT_TRUE(py_path_ob); -#else - py::owned_ref py_path_ob = py::to_object("/tmp/"); -#endif int eq = PyObject_RichCompareBool(ob.get(), py_path_ob.get(), Py_EQ); EXPECT_EQ(eq, 1); } From 0dc40054c4447d20043e59bf148d1db3e6796831 Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Mon, 6 Jul 2020 14:12:51 -0400 Subject: [PATCH 07/15] remove more pre 3.5 compat --- include/libpy/to_object.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/include/libpy/to_object.h b/include/libpy/to_object.h index d9be3b28..61e526c8 100644 --- a/include/libpy/to_object.h +++ b/include/libpy/to_object.h @@ -142,7 +142,7 @@ struct to_object { static py::owned_ref<> f(const std::filesystem::path& path) { py::owned_ref path_str{ PyUnicode_FromStringAndSize(path.c_str(), path.native().length())}; -#if PY_VERSION_HEX >= 0x03040000 + py::owned_ref pathlib(PyImport_ImportModule("pathlib")); if (!pathlib) { throw py::exception(); @@ -158,9 +158,6 @@ struct to_object { throw py::exception(); } return path_obj; -#else - return path_str; -#endif } }; From 411089224bbd5ff5b07873dc4605f4e91e5e1def Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Mon, 6 Jul 2020 14:29:23 -0400 Subject: [PATCH 08/15] try grabbing the coredump --- .github/workflows/main.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1e4da0eb..507863bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,9 +84,22 @@ jobs: run: | echo ::set-env name=CC::gcc-8 echo ::set-env name=CXX::g++-8 - - name: Run the tests + - name: Run the tests (osx) + if: startsWith(matrix.os, 'macos') + run: | + ulimit -c unlimited + sudo make -j2 test + - name: Run the tests (ubuntu) + if: startsWith(matrix.os, 'ubuntu') run: | - make -j2 test + ulimit -c unlimited + sudo chmod -R +rwx /var/crash + sudo make -j2 test + - uses: actions/upload-artifact@master # capture all crashes as build artifacts + with: + name: crashes + path: /var/crash + - name: Check that we can still install run: | pip install . From ca1d3d0e283dc28479ae8f8eff439900e9de472f Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Mon, 6 Jul 2020 14:36:20 -0400 Subject: [PATCH 09/15] do not sudo tests --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 507863bc..47775cc1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,13 +88,13 @@ jobs: if: startsWith(matrix.os, 'macos') run: | ulimit -c unlimited - sudo make -j2 test + make -j2 test - name: Run the tests (ubuntu) if: startsWith(matrix.os, 'ubuntu') run: | ulimit -c unlimited sudo chmod -R +rwx /var/crash - sudo make -j2 test + make -j2 test - uses: actions/upload-artifact@master # capture all crashes as build artifacts with: name: crashes From f8e872c72e0427efebc337871f382a0669b08cd5 Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Mon, 6 Jul 2020 14:48:42 -0400 Subject: [PATCH 10/15] upload on failure --- .github/workflows/main.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 47775cc1..21e0e65c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -85,20 +85,18 @@ jobs: echo ::set-env name=CC::gcc-8 echo ::set-env name=CXX::g++-8 - name: Run the tests (osx) - if: startsWith(matrix.os, 'macos') - run: | - ulimit -c unlimited - make -j2 test - - name: Run the tests (ubuntu) if: startsWith(matrix.os, 'ubuntu') run: | - ulimit -c unlimited sudo chmod -R +rwx /var/crash + - name: Run the tests + run: | + ulimit -c unlimited make -j2 test - - uses: actions/upload-artifact@master # capture all crashes as build artifacts + - uses: actions/upload-artifact@v2 # capture all crashes as build artifacts + if: failure() with: name: crashes - path: /var/crash + path: /var/crash/ - name: Check that we can still install run: | From 4a4515e63badb6174e30d7be6cbbaf30147bd73c Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Mon, 6 Jul 2020 15:49:41 -0400 Subject: [PATCH 11/15] run apport-retrace --- .github/workflows/main.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21e0e65c..e9d7e785 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,14 +84,18 @@ jobs: run: | echo ::set-env name=CC::gcc-8 echo ::set-env name=CXX::g++-8 - - name: Run the tests (osx) - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo chmod -R +rwx /var/crash + # - name: Write permissions to ubunu + # if: startsWith(matrix.os, 'ubuntu') + # run: | + # sudo chmod -R +rwx /var/crash - name: Run the tests run: | ulimit -c unlimited make -j2 test + - name: Run apport-retrace if we've failed + if: startsWith(matrix.os, 'ubuntu') && failure() + run: | + apport-retrace /var/crash/*.crashes - uses: actions/upload-artifact@v2 # capture all crashes as build artifacts if: failure() with: From fd1383ce012457193bd2dc976f96fed77e5250c0 Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Tue, 7 Jul 2020 14:53:38 -0400 Subject: [PATCH 12/15] revert a bunch of changes, only use gcc-9 --- .github/workflows/main.yml | 19 ++----------------- etc/Makefile.buildkite | 7 ------- 2 files changed, 2 insertions(+), 24 deletions(-) delete mode 100644 etc/Makefile.buildkite diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e9d7e785..f76bc336 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,26 +82,11 @@ jobs: - name: Set gcc envvars if: matrix.compiler == 'gcc' run: | - echo ::set-env name=CC::gcc-8 - echo ::set-env name=CXX::g++-8 - # - name: Write permissions to ubunu - # if: startsWith(matrix.os, 'ubuntu') - # run: | - # sudo chmod -R +rwx /var/crash + echo ::set-env name=CC::gcc-9 + echo ::set-env name=CXX::g++-9 - name: Run the tests run: | - ulimit -c unlimited make -j2 test - - name: Run apport-retrace if we've failed - if: startsWith(matrix.os, 'ubuntu') && failure() - run: | - apport-retrace /var/crash/*.crashes - - uses: actions/upload-artifact@v2 # capture all crashes as build artifacts - if: failure() - with: - name: crashes - path: /var/crash/ - - name: Check that we can still install run: | pip install . diff --git a/etc/Makefile.buildkite b/etc/Makefile.buildkite deleted file mode 100644 index 2184ab17..00000000 --- a/etc/Makefile.buildkite +++ /dev/null @@ -1,7 +0,0 @@ -CXX := g++-8 -CC := gcc-8 -GCOV := gcov-8 -AR := gcc-ar-8 -ASAN_SYMBOLIZER_PATH := llvm-symbolizer-6 -CLANG_TIDY := clang-tidy-6 -CLANG_FORMAT := clang-format-6 From e7b6159681409ae688bb9b44c6fe7cb167dd62a0 Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Tue, 7 Jul 2020 15:45:11 -0400 Subject: [PATCH 13/15] DOC: update documentation for gcc --- README.rst | 2 +- docs/source/install.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6fa83036..93a1ae06 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ libpy supports: libpy requires: -- gcc>=8 or clang>=10 +- gcc>=9 or clang>=10 - numpy>=1.11.3 Optional Requirements diff --git a/docs/source/install.rst b/docs/source/install.rst index 0381e361..78ae57b6 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -11,7 +11,7 @@ lipy supports: lipy requires: -- gcc>=8 or clang>=10 +- gcc>=9 or clang>=10 - numpy>=1.11.3 Optional Requirements From d2960493bb3bb394db1ba4b6217842cf1285c782 Mon Sep 17 00:00:00 2001 From: Gerry Manoim Date: Thu, 9 Jul 2020 23:07:27 -0400 Subject: [PATCH 14/15] BUG: Add ld_flags.py to manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index e403c0d0..ac2509f0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,7 @@ include etc/detect-compiler.cc include etc/build-and-run include etc/ext_suffix.py include etc/asan-path +include etc/ld_flags.py include etc/python_version.py include version recursive-include src/ *.cc From 70d76c6d9714768904e32be9fb87a816e284545f Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Thu, 9 Jul 2020 23:08:37 -0400 Subject: [PATCH 15/15] BLD: attempt to install from sdist in tests --- .github/workflows/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f76bc336..2dc0191a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,9 +87,10 @@ jobs: - name: Run the tests run: | make -j2 test - - name: Check that we can still install + - name: Build and install from an sdist run: | - pip install . + python setup.py sdist + pip install dist/libpy-*.tar.gz - name: Check that docs can be built # note we don't build this on macOS because the GHA python is built with # the wrong version of macOS, confusing clang.