From 66289e76c1dec64be43579c91a245629850af925 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Wed, 7 Aug 2024 12:02:17 +0200 Subject: [PATCH 01/25] Bump to 6.0.0 for devel. --- CMakeLists.txt | 4 ++-- doc/install.rst | 2 +- tools/gha_manylinux.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 32203ad3..6dbb7525 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ if(NOT CMAKE_BUILD_TYPE) FORCE) endif() -project(heyoka.py VERSION 5.1.0 LANGUAGES CXX C) +project(heyoka.py VERSION 6.0.0 LANGUAGES CXX C) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/yacma") @@ -118,7 +118,7 @@ find_package(fmt REQUIRED CONFIG) message(STATUS "fmt version: ${fmt_VERSION}") # heyoka. -find_package(heyoka 5.1.0 REQUIRED CONFIG) +find_package(heyoka 6.0.0 REQUIRED CONFIG) # Python. diff --git a/doc/install.rst b/doc/install.rst index ade3e401..979b2d3f 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -9,7 +9,7 @@ Dependencies heyoka.py has several Python and C++ dependencies. On the C++ side, heyoka.py depends on: * the `heyoka C++ library `__, - version 5.1.x (**mandatory**), + version 6.0.x (**mandatory**), * the `Boost `__ C++ libraries (**mandatory**), * the `{fmt} `__ library (**mandatory**), * the `TBB `__ library (**mandatory**), diff --git a/tools/gha_manylinux.sh b/tools/gha_manylinux.sh index d438dafb..c33c927f 100644 --- a/tools/gha_manylinux.sh +++ b/tools/gha_manylinux.sh @@ -42,7 +42,7 @@ else fi # The heyoka version to be used for releases. -export HEYOKA_VERSION_RELEASE="5.1.0" +export HEYOKA_VERSION_RELEASE="6.0.0" # Check if this is a release build. if [[ "${GITHUB_REF}" == "refs/tags/v"* ]]; then From 5b59d07cd55fe77ea2969eea5bf0966413ef8003 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 10:11:12 +0200 Subject: [PATCH 02/25] Adapt for recent heyoka changes. - add the code_model enum, - add the code_model and parjit LLVM kwargs, - expose llvm_multi_state, - adapt the testing code. --- heyoka/_test_cfunc.py | 23 ++++----- heyoka/_test_mp.py | 2 +- heyoka/cfunc.cpp | 68 +++++++++++++++------------ heyoka/common_utils.hpp | 22 ++++++++- heyoka/core.cpp | 53 ++++++++++++++++++++- heyoka/expose_batch_integrators.cpp | 17 ++++--- heyoka/expose_sgp4_propagators.cpp | 72 +++++++++++++++-------------- heyoka/taylor_expose_integrator.cpp | 10 ++-- 8 files changed, 175 insertions(+), 92 deletions(-) diff --git a/heyoka/_test_cfunc.py b/heyoka/_test_cfunc.py index 81346f64..191a5ce4 100644 --- a/heyoka/_test_cfunc.py +++ b/heyoka/_test_cfunc.py @@ -21,8 +21,8 @@ def test_basic(self): x, y, z, s = make_vars("x", "y", "z", "s") cf = cfunc([y * (x + z)], [x, y, z]) - self.assertFalse(cf.llvm_state_scalar.force_avx512) - self.assertFalse(cf.llvm_state_scalar.slp_vectorize) + self.assertTrue(all(not _.force_avx512 for _ in cf.llvm_states)) + self.assertTrue(all(not _.slp_vectorize for _ in cf.llvm_states)) self.assertEqual(cf([1, 2, 3]), copy(cf)([1, 2, 3])) self.assertEqual(cf([1, 2, 3]), deepcopy(cf)([1, 2, 3])) @@ -31,32 +31,27 @@ def test_basic(self): self.assertEqual(cf.vars, [x, y, z]) self.assertEqual(cf.fn, [y * (x + z)]) self.assertEqual(len(cf.dc), 6) - self.assertNotEqual(len(cf.llvm_state_scalar.get_ir()), 0) + self.assertTrue(all(len(_.get_ir()) != 0 for _ in cf.llvm_states)) self.assertEqual(deepcopy(cf).vars, [x, y, z]) self.assertEqual(deepcopy(cf).fn, [y * (x + z)]) self.assertEqual(deepcopy(cf).dc, cf.dc) self.assertEqual( - deepcopy(cf).llvm_state_scalar.get_ir(), cf.llvm_state_scalar.get_ir() - ) - self.assertEqual( - deepcopy(cf).llvm_state_scalar_s.get_ir(), cf.llvm_state_scalar_s.get_ir() - ) - self.assertEqual( - deepcopy(cf).llvm_state_batch_s.get_ir(), cf.llvm_state_batch_s.get_ir() + [_.get_ir() for _ in deepcopy(cf).llvm_states], + [_.get_ir() for _ in cf.llvm_states], ) self.assertEqual(pickle.loads(pickle.dumps(cf)).vars, [x, y, z]) self.assertEqual(pickle.loads(pickle.dumps(cf)).fn, [y * (x + z)]) self.assertEqual(pickle.loads(pickle.dumps(cf)).dc, cf.dc) self.assertEqual( - pickle.loads(pickle.dumps(cf)).llvm_state_scalar.get_ir(), - cf.llvm_state_scalar.get_ir(), + [_.get_ir() for _ in pickle.loads(pickle.dumps(cf)).llvm_states], + [_.get_ir() for _ in cf.llvm_states], ) cf = cfunc([y * (x + z)], vars=[y, z, x], force_avx512=True, slp_vectorize=True) self.assertEqual(cf.vars, [y, z, x]) - self.assertTrue(cf.llvm_state_scalar.force_avx512) - self.assertTrue(cf.llvm_state_scalar.slp_vectorize) + self.assertTrue(all(_.force_avx512 for _ in cf.llvm_states)) + self.assertTrue(all(_.slp_vectorize for _ in cf.llvm_states)) # Tests for correct detection of number of params, time dependency # and list of variables. diff --git a/heyoka/_test_mp.py b/heyoka/_test_mp.py index 3ab868c6..2e7f633e 100644 --- a/heyoka/_test_mp.py +++ b/heyoka/_test_mp.py @@ -32,7 +32,7 @@ def test_cfunc(self): with self.assertRaises(ValueError) as cm: cfunc(func, vars=[y, x], fp_type=real, prec=-1) self.assertTrue( - "An invalid precision value of -1 was passed to add_cfunc()" + "An invalid precision value of -1 was passed to make_multi_cfunc()" in str(cm.exception) ) diff --git a/heyoka/cfunc.cpp b/heyoka/cfunc.cpp index c7a9e6a2..d0a6830e 100644 --- a/heyoka/cfunc.cpp +++ b/heyoka/cfunc.cpp @@ -8,10 +8,12 @@ #include +#include #include #include #include #include +#include #include #include #include @@ -29,6 +31,7 @@ #include #include +#include #if defined(HEYOKA_HAVE_REAL128) @@ -82,33 +85,33 @@ void expose_add_cfunc_impl(py::module &m, const char *suffix) namespace kw = hey::kw; py::class_> cfunc_inst(m, fmt::format("cfunc_{}", suffix).c_str(), py::dynamic_attr{}); - cfunc_inst.def(py::init([](std::vector fn, std::vector vars, bool high_accuracy, - bool compact_mode, bool parallel_mode, std::uint32_t batch_size, long long prec, - unsigned opt_level, bool force_avx512, bool slp_vectorize, bool fast_math) { - // Forbid batch sizes > 1 for everything but double and float. - // NOTE: there is a similar check on the C++ side regarding mppp::real, but in Python - // specifically we want to be pragmatic and allow for batch operations only if we know that it - // makes sense performance-wise (and we also want to avoid buggy batch operations on long - // double). - if (!std::is_same_v && !std::is_same_v && batch_size > 1u) [[unlikely]] { - py_throw(PyExc_ValueError, - "Batch sizes greater than 1 are not supported for this floating-point type"); - } - - // NOTE: release the GIL during compilation. - py::gil_scoped_release release; - - return hey::cfunc{std::move(fn), std::move(vars), kw::high_accuracy = high_accuracy, - kw::compact_mode = compact_mode, kw::parallel_mode = parallel_mode, - kw::opt_level = opt_level, kw::force_avx512 = force_avx512, - kw::slp_vectorize = slp_vectorize, kw::batch_size = batch_size, - kw::fast_math = fast_math, kw::prec = prec, - // NOTE: it is important to disable the prec checking - // here as we will have our own custom implementation - // of precision checking to deal with NumPy arrays. - kw::check_prec = false}; - }), - "fn"_a, "vars"_a, HEYOKA_PY_CFUNC_ARGS(default_cm), HEYOKA_PY_LLVM_STATE_ARGS); + cfunc_inst.def( + py::init([](std::vector fn, std::vector vars, bool high_accuracy, + bool compact_mode, bool parallel_mode, std::uint32_t batch_size, long long prec, unsigned opt_level, + bool force_avx512, bool slp_vectorize, bool fast_math, hey::code_model code_model, bool parjit) { + // Forbid batch sizes > 1 for everything but double and float. + // NOTE: there is a similar check on the C++ side regarding mppp::real, but in Python + // specifically we want to be pragmatic and allow for batch operations only if we know that it + // makes sense performance-wise (and we also want to avoid buggy batch operations on long + // double). + if (!std::is_same_v && !std::is_same_v && batch_size > 1u) [[unlikely]] { + py_throw(PyExc_ValueError, "Batch sizes greater than 1 are not supported for this floating-point type"); + } + + // NOTE: release the GIL during compilation. + py::gil_scoped_release release; + + return hey::cfunc{std::move(fn), std::move(vars), kw::high_accuracy = high_accuracy, + kw::compact_mode = compact_mode, kw::parallel_mode = parallel_mode, + kw::opt_level = opt_level, kw::force_avx512 = force_avx512, + kw::slp_vectorize = slp_vectorize, kw::batch_size = batch_size, + kw::fast_math = fast_math, kw::prec = prec, + // NOTE: it is important to disable the prec checking + // here as we will have our own custom implementation + // of precision checking to deal with NumPy arrays. + kw::check_prec = false, kw::code_model = code_model, kw::parjit = parjit}; + }), + "fn"_a, "vars"_a, HEYOKA_PY_CFUNC_ARGS(default_cm), HEYOKA_PY_LLVM_STATE_ARGS); // Typedefs for the call operator. using array_or_iter_t = std::variant; @@ -459,9 +462,14 @@ void expose_add_cfunc_impl(py::module &m, const char *suffix) cfunc_inst.def_property_readonly("fn", &hey::cfunc::get_fn); cfunc_inst.def_property_readonly("vars", &hey::cfunc::get_vars); cfunc_inst.def_property_readonly("dc", &hey::cfunc::get_dc); - cfunc_inst.def_property_readonly("llvm_state_scalar", &hey::cfunc::get_llvm_state_scalar); - cfunc_inst.def_property_readonly("llvm_state_scalar_s", &hey::cfunc::get_llvm_state_scalar_s); - cfunc_inst.def_property_readonly("llvm_state_batch_s", &hey::cfunc::get_llvm_state_batch_s); + cfunc_inst.def_property_readonly("llvm_states", [](const hey::cfunc &self) { + const auto &st = self.get_llvm_states(); + + using ret_t = std::variant>, + std::reference_wrapper>; + + return std::visit([](const auto &v) -> ret_t { return std::cref(v); }, st); + }); cfunc_inst.def_property_readonly("high_accuracy", &hey::cfunc::get_high_accuracy); cfunc_inst.def_property_readonly("compact_mode", &hey::cfunc::get_compact_mode); cfunc_inst.def_property_readonly("parallel_mode", &hey::cfunc::get_parallel_mode); diff --git a/heyoka/common_utils.hpp b/heyoka/common_utils.hpp index dfa295f5..e552d7fa 100644 --- a/heyoka/common_utils.hpp +++ b/heyoka/common_utils.hpp @@ -11,8 +11,10 @@ #include #include +#include #include #include +#include #if defined(__GLIBCXX__) @@ -22,10 +24,12 @@ #include #include +#include #include #include +#include #include // NOTE: implementation of Py_SET_TYPE() for Python < 3.9. See: @@ -56,6 +60,21 @@ std::string str(const py::handle &); bool callable(const py::handle &); +// Helper to expose the llvm_state getter +// for a Taylor integrator. +template +inline void expose_llvm_state_property_ta(py::class_ &c) +{ + c.def_property_readonly("llvm_state", [](const T &self) { + const auto &st = self.get_llvm_state(); + + using ret_t = std::variant, + std::reference_wrapper>; + + return std::visit([](const auto &v) -> ret_t { return std::cref(v); }, st); + }); +} + // Helper to expose the llvm_state getter // for a generic object. template @@ -217,7 +236,8 @@ py::array as_carray(const py::iterable &, int); // Macros to avoid repetitions in commonly-used keyword arguments. #define HEYOKA_PY_LLVM_STATE_ARGS \ "opt_level"_a.noconvert() = 3, "force_avx512"_a.noconvert() = false, "slp_vectorize"_a.noconvert() = false, \ - "fast_math"_a.noconvert() = false + "fast_math"_a.noconvert() = false, "code_model"_a.noconvert() = heyoka::code_model::small, \ + "parjit"_a.noconvert() = heyoka::detail::default_parjit #define HEYOKA_PY_CFUNC_ARGS(default_cm) \ "high_accuracy"_a.noconvert() = false, "compact_mode"_a.noconvert() = default_cm, \ diff --git a/heyoka/core.cpp b/heyoka/core.cpp index 93858165..58085e17 100644 --- a/heyoka/core.cpp +++ b/heyoka/core.cpp @@ -164,16 +164,24 @@ PYBIND11_MODULE(core, m) } }); + // code_model enum. + py::enum_(m, "code_model") + .value("tiny", hey::code_model::tiny) + .value("small", hey::code_model::small) + .value("kernel", hey::code_model::kernel) + .value("medium", hey::code_model::medium) + .value("large", hey::code_model::large); + // LLVM state. py::class_(m, "llvm_state", py::dynamic_attr{}) .def("get_ir", &hey::llvm_state::get_ir) .def("get_bc", [](const hey::llvm_state &s) { return py::bytes(s.get_bc()); }) .def("get_object_code", [](const hey::llvm_state &s) { return py::bytes(s.get_object_code()); }) - // NOTE: make these read-only for the moment. .def_property_readonly("opt_level", &hey::llvm_state::get_opt_level) .def_property_readonly("fast_math", [](const hey::llvm_state &s) { return s.fast_math(); }) .def_property_readonly("force_avx512", [](const hey::llvm_state &s) { return s.force_avx512(); }) .def_property_readonly("slp_vectorize", &hey::llvm_state::get_slp_vectorize) + .def_property_readonly("code_model", &hey::llvm_state::get_code_model) // Repr. .def("__repr__", [](const hey::llvm_state &s) { @@ -195,6 +203,49 @@ PYBIND11_MODULE(core, m) [](const py::object &, std::size_t limit) { hey::llvm_state::set_memcache_limit(limit); }) .def_static("clear_memcache", &hey::llvm_state::clear_memcache); + // LLVM multi state. + py::class_(m, "llvm_multi_state", py::dynamic_attr{}) + .def("get_ir", &hey::llvm_multi_state::get_ir) + .def("get_bc", + [](const hey::llvm_multi_state &s) { + py::list ret; + + for (const auto &cur_bc : s.get_bc()) { + ret.append(py::bytes(cur_bc)); + } + + return ret; + }) + .def("get_object_code", + [](const hey::llvm_multi_state &s) { + py::list ret; + + for (const auto &cur_ob : s.get_object_code()) { + ret.append(py::bytes(cur_ob)); + } + + return ret; + }) + .def_property_readonly("opt_level", &hey::llvm_multi_state::get_opt_level) + .def_property_readonly("fast_math", [](const hey::llvm_multi_state &s) { return s.fast_math(); }) + .def_property_readonly("force_avx512", [](const hey::llvm_multi_state &s) { return s.force_avx512(); }) + .def_property_readonly("slp_vectorize", &hey::llvm_multi_state::get_slp_vectorize) + .def_property_readonly("parjit", &hey::llvm_multi_state::get_parjit) + .def_property_readonly("code_model", &hey::llvm_multi_state::get_code_model) + // Repr. + .def("__repr__", + [](const hey::llvm_multi_state &s) { + std::ostringstream oss; + oss << s; + return oss.str(); + }) + // Copy/deepcopy. + .def("__copy__", heypy::copy_wrapper) + .def("__deepcopy__", heypy::deepcopy_wrapper, "memo"_a) + // Pickle support. + .def(py::pickle(&heypy::pickle_getstate_wrapper, + &heypy::pickle_setstate_wrapper)); + // Expression. heypy::expose_expression(m); diff --git a/heyoka/expose_batch_integrators.cpp b/heyoka/expose_batch_integrators.cpp index 952d4c82..370036cc 100644 --- a/heyoka/expose_batch_integrators.cpp +++ b/heyoka/expose_batch_integrators.cpp @@ -92,7 +92,7 @@ void expose_batch_integrator_impl(py::module_ &m, const std::string &suffix) std::optional time_ob, std::optional pars_ob, T tol, bool high_accuracy, bool compact_mode, std::vector tes, std::vector ntes, bool parallel_mode, unsigned opt_level, bool force_avx512, bool slp_vectorize, - bool fast_math) { + bool fast_math, hey::code_model code_model, bool parjit) { // Fetch the dtype corresponding to T. const auto dt = get_dtype(); @@ -176,7 +176,9 @@ void expose_batch_integrator_impl(py::module_ &m, const std::string &suffix) kw::opt_level = opt_level, kw::force_avx512 = force_avx512, kw::slp_vectorize = slp_vectorize, - kw::fast_math = fast_math}; + kw::fast_math = fast_math, + kw::code_model = code_model, + kw::parjit = parjit}; } else { // Times not provided. @@ -199,7 +201,9 @@ void expose_batch_integrator_impl(py::module_ &m, const std::string &suffix) kw::opt_level = opt_level, kw::force_avx512 = force_avx512, kw::slp_vectorize = slp_vectorize, - kw::fast_math = fast_math}; + kw::fast_math = fast_math, + kw::code_model = code_model, + kw::parjit = parjit}; } }, vsys); @@ -213,10 +217,11 @@ void expose_batch_integrator_impl(py::module_ &m, const std::string &suffix) std::optional time, std::optional pars, T tol, bool high_accuracy, bool compact_mode, std::vector tes, std::vector ntes, bool parallel_mode, unsigned opt_level, - bool force_avx512, bool slp_vectorize, bool fast_math) { + bool force_avx512, bool slp_vectorize, bool fast_math, hey::code_model code_model, + bool parjit) { return tab_ctor_impl(std::move(vsys), state, std::move(time), std::move(pars), tol, high_accuracy, compact_mode, std::move(tes), std::move(ntes), parallel_mode, opt_level, - force_avx512, slp_vectorize, fast_math); + force_avx512, slp_vectorize, fast_math, code_model, parjit); }), "sys"_a, "state"_a, "time"_a = py::none{}, "pars"_a = py::none{}, "tol"_a.noconvert() = static_cast(0), "high_accuracy"_a = false, "compact_mode"_a = false, "t_events"_a = py::list{}, "nt_events"_a = py::list{}, @@ -658,7 +663,7 @@ void expose_batch_integrator_impl(py::module_ &m, const std::string &suffix) &pickle_setstate_wrapper>)); // Expose the llvm state getter. - expose_llvm_state_property(tab_c); + expose_llvm_state_property_ta(tab_c); } } // namespace diff --git a/heyoka/expose_sgp4_propagators.cpp b/heyoka/expose_sgp4_propagators.cpp index b4d73a79..ce0a18e7 100644 --- a/heyoka/expose_sgp4_propagators.cpp +++ b/heyoka/expose_sgp4_propagators.cpp @@ -106,42 +106,44 @@ void expose_sgp4_propagator_impl(py::module_ &m, const std::string &suffix) py::class_ prop_cl(m, fmt::format("_model_sgp4_propagator_{}", suffix).c_str(), py::dynamic_attr{}, docstrings::sgp4_propagator(std::same_as ? "double" : "single").c_str()); - prop_cl.def(py::init([](py::list sat_list, std::uint32_t diff_order, bool high_accuracy, bool compact_mode, - bool parallel_mode, std::uint32_t batch_size, long long, unsigned opt_level, - bool force_avx512, bool slp_vectorize, bool fast_math) { - // Check that the sgp4 module is available. - try { - py::module_::import("sgp4.api"); - } catch (...) { - py_throw( - PyExc_ImportError, - "The Python module 'sgp4' must be installed in order to be able to build sgp4 propagators"); - } + prop_cl.def( + py::init([](py::list sat_list, std::uint32_t diff_order, bool high_accuracy, bool compact_mode, + bool parallel_mode, std::uint32_t batch_size, long long, unsigned opt_level, bool force_avx512, + bool slp_vectorize, bool fast_math, hy::code_model code_model, bool parjit) { + // Check that the sgp4 module is available. + try { + py::module_::import("sgp4.api"); + } catch (...) { + py_throw(PyExc_ImportError, + "The Python module 'sgp4' must be installed in order to be able to build sgp4 propagators"); + } + + // Turn sat_list into a data vector. + const auto sat_data = sat_list_to_vector(sat_list); + assert(sat_data.size() % 9u == 0u); + + // Create the input span for the constructor. + using span_t = hy::mdspan>; + const span_t in(sat_data.data(), boost::numeric_cast(sat_data.size()) / 9u); - // Turn sat_list into a data vector. - const auto sat_data = sat_list_to_vector(sat_list); - assert(sat_data.size() % 9u == 0u); - - // Create the input span for the constructor. - using span_t = hy::mdspan>; - const span_t in(sat_data.data(), boost::numeric_cast(sat_data.size()) / 9u); - - // NOTE: release the GIL during compilation. - py::gil_scoped_release release; - - return prop_t{in, - kw::diff_order = diff_order, - kw::high_accuracy = high_accuracy, - kw::compact_mode = compact_mode, - kw::parallel_mode = parallel_mode, - kw::batch_size = batch_size, - kw::opt_level = opt_level, - kw::force_avx512 = force_avx512, - kw::slp_vectorize = slp_vectorize, - kw::fast_math = fast_math}; - }), - "sat_list"_a.noconvert(), "diff_order"_a.noconvert() = static_cast(0), - HEYOKA_PY_CFUNC_ARGS(false), HEYOKA_PY_LLVM_STATE_ARGS, docstrings::sgp4_propagator_init().c_str()); + // NOTE: release the GIL during compilation. + py::gil_scoped_release release; + + return prop_t{in, + kw::diff_order = diff_order, + kw::high_accuracy = high_accuracy, + kw::compact_mode = compact_mode, + kw::parallel_mode = parallel_mode, + kw::batch_size = batch_size, + kw::opt_level = opt_level, + kw::force_avx512 = force_avx512, + kw::slp_vectorize = slp_vectorize, + kw::fast_math = fast_math, + kw::code_model = code_model, + kw::parjit = parjit}; + }), + "sat_list"_a.noconvert(), "diff_order"_a.noconvert() = static_cast(0), + HEYOKA_PY_CFUNC_ARGS(false), HEYOKA_PY_LLVM_STATE_ARGS, docstrings::sgp4_propagator_init().c_str()); prop_cl.def_property_readonly( "jdtype", [](const prop_t &) -> py::object { diff --git a/heyoka/taylor_expose_integrator.cpp b/heyoka/taylor_expose_integrator.cpp index ac214c18..000f0c70 100644 --- a/heyoka/taylor_expose_integrator.cpp +++ b/heyoka/taylor_expose_integrator.cpp @@ -108,7 +108,7 @@ template void expose_taylor_integrator_impl(py::module &m, const std::string &suffix) { using namespace py::literals; - namespace kw = heyoka::kw; + namespace kw = hey::kw; using t_ev_t = hey::t_event; using nt_ev_t = hey::nt_event; @@ -119,7 +119,7 @@ void expose_taylor_integrator_impl(py::module &m, const std::string &suffix) cl.def(py::init([](std::variant vsys, std::vector state, T time, std::vector pars, T tol, bool high_accuracy, bool compact_mode, std::vector tes, std::vector ntes, bool parallel_mode, unsigned opt_level, bool force_avx512, bool slp_vectorize, bool fast_math, - long long prec) { + hey::code_model code_model, bool parjit, long long prec) { // NOTE: GIL release is fine here even if the events contain // Python objects, as the event vectors are moved in // upon construction and thus we should never end up calling @@ -142,7 +142,9 @@ void expose_taylor_integrator_impl(py::module &m, const std::string &suffix) kw::force_avx512 = force_avx512, kw::slp_vectorize = slp_vectorize, kw::fast_math = fast_math, - kw::prec = prec}; + kw::prec = prec, + kw::code_model = code_model, + kw::parjit = parjit}; }, vsys); }), @@ -480,7 +482,7 @@ void expose_taylor_integrator_impl(py::module &m, const std::string &suffix) .def_property_readonly("nt_events", &hey::taylor_adaptive::get_nt_events); // Expose the llvm state getter. - expose_llvm_state_property(cl); + expose_llvm_state_property_ta(cl); #if defined(HEYOKA_HAVE_REAL) From 373476df504fa301447496c7b1329f5539635939 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 10:41:48 +0200 Subject: [PATCH 03/25] Fix test for latest sympy changes. --- heyoka/_test_sympy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heyoka/_test_sympy.py b/heyoka/_test_sympy.py index 2b0510f5..9b45c30d 100644 --- a/heyoka/_test_sympy.py +++ b/heyoka/_test_sympy.py @@ -267,7 +267,7 @@ def test_func_conversion(self): self.assertEqual(to_sympy(hx + hy + hz), x + y + z) self.assertEqual(to_sympy(hsum([hx, hy, hz])), x + y + z) self.assertEqual(to_sympy(hsum([hx])), x) - self.assertEqual(to_sympy(hsum([])), 0.0) + self.assertEqual(to_sympy(hsum([])), 0) self.assertEqual( hsum([ha, hb, hc, hx, hy, hz]), from_sympy(x + y + z + a + b + c) ) From 4be324f58513f66b967fd91b207522039f1c80f9 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 11:45:50 +0200 Subject: [PATCH 04/25] Try changing Python version on Windows. --- .github/workflows/gh_actions_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh_actions_ci.yml b/.github/workflows/gh_actions_ci.yml index 7d733123..a50ec397 100644 --- a/.github/workflows/gh_actions_ci.yml +++ b/.github/workflows/gh_actions_ci.yml @@ -118,14 +118,14 @@ jobs: - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true - python-version: "3.10" + python-version: "3.12" channels: conda-forge channel-priority: strict - uses: ilammy/msvc-dev-cmd@v1 - name: Build shell: pwsh run: | - conda install -y python=3.10 git pybind11 numpy<2 cmake llvmdev tbb-devel tbb astroquery libboost-devel fmt<11 skyfield spdlog sleef sympy cloudpickle zlib libzlib 'mppp=1.*' numba + conda install -y python=3.12 git pybind11 numpy<2 cmake llvmdev tbb-devel tbb astroquery libboost-devel fmt<11 skyfield spdlog sleef sympy cloudpickle zlib libzlib 'mppp=1.*' numba git clone --depth 1 https://github.com/bluescarni/heyoka.git heyoka_cpp cd heyoka_cpp mkdir build From a8b1e9c72365c8155d1a66afd4b4442f3c095c4a Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 11:47:56 +0200 Subject: [PATCH 05/25] Try setting a longer timeout for notebook execution. --- doc/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/conf.py b/doc/conf.py index 6e942bb1..92440ea8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -111,6 +111,9 @@ # Force printing traceback to stderr on execution error. nb_execution_show_tb = True +# Set a longer timeout for notebook execution. +nb_execution_timeout = 120 + latex_engine = "xelatex" myst_enable_extensions = [ From 801170a4001feb0cd731298795dd34a82f612e81 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 12:51:24 +0200 Subject: [PATCH 06/25] Tentative windows debugging. --- heyoka/core.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/heyoka/core.cpp b/heyoka/core.cpp index 58085e17..6a9b16f1 100644 --- a/heyoka/core.cpp +++ b/heyoka/core.cpp @@ -396,6 +396,24 @@ PYBIND11_MODULE(core, m) #endif heypy::detail::tbb_gc.reset(); })); + + std::set_terminate([]() { + try { + std::exception_ptr eptr{std::current_exception()}; + if (eptr) { + std::rethrow_exception(eptr); + } else { + std::cerr << "Exiting without exception\n"; + } + } catch (const std::exception &ex) { + std::cerr << "Exception: " << ex.what() << std::endl; + std::cout << "Exception: " << ex.what() << std::endl; + } catch (...) { + std::cerr << "Unknown exception caught" << std::endl; + std::cout << "Unknown exception caught" << std::endl; + } + std::exit(EXIT_FAILURE); + }); } #if defined(__clang__) From ecf7e80c2f53b01c16864712fae4a307ec56e862 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 13:06:06 +0200 Subject: [PATCH 07/25] Fix section headings in the map inversion notebook. --- doc/notebooks/map_inversion.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/notebooks/map_inversion.ipynb b/doc/notebooks/map_inversion.ipynb index 5d6093f3..593449bf 100644 --- a/doc/notebooks/map_inversion.ipynb +++ b/doc/notebooks/map_inversion.ipynb @@ -461,7 +461,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# The map inversion algorithm\n", + "## The map inversion algorithm\n", "\n", "We need to write a few helper function as to be able to perform the map inversion in **heyoka.py**. Fistly we must be able to extract the symbolic epressions of the polynomials $\\mathcal P^n_\\circ$ from the various states of the variational equations. \n", "\n", @@ -738,7 +738,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Let's visualize the impact points\n", + "## Let's visualize the impact points\n", "\n", "Our final aim was to compute the variations of the final state at the event, so we now produce 100 random **initial conditions** variations, and we compute, via our Taylor maps, the corresponding final conditions, which we expect all to lie on the sphere:" ] @@ -824,7 +824,7 @@ ], "metadata": { "kernelspec": { - "display_name": "heyokapy_devel", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -842,5 +842,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } From c0afff8a40d841851aa2ee581c5bbd506356505b Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 13:12:11 +0200 Subject: [PATCH 08/25] A couple of clarificatory remarks on quad precision support in the tutorials. --- doc/notebooks/The expression system.ipynb | 4 ++-- doc/notebooks/ext_precision.ipynb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/notebooks/The expression system.ipynb b/doc/notebooks/The expression system.ipynb index 731fe5ca..67abf0dd 100644 --- a/doc/notebooks/The expression system.ipynb +++ b/doc/notebooks/The expression system.ipynb @@ -107,7 +107,7 @@ "Note that, while single and double precision are always supported in heyoka.py, support for [extended-precision](<./ext_precision.ipynb>) floating-point types varies depending on the software/hardware platform. Specifically:\n", "\n", "- on x86 processors, the NumPy {py:class}`~numpy.longdouble` type corresponds to 80-bit extended precision on most platforms (the exception being MSVC on Windows, where ``longdouble == float``);\n", - "- on some platforms (e.g., Unix ARM 64), the {py:class}`~numpy.longdouble` type implements the IEEE [quadruple-precision floating-point format](https://en.wikipedia.org/wiki/Quadruple-precision_floating-point_format);\n", + "- on some platforms (e.g., Linux ARM 64), the {py:class}`~numpy.longdouble` type implements the IEEE [quadruple-precision floating-point format](https://en.wikipedia.org/wiki/Quadruple-precision_floating-point_format);\n", "- on some platforms where {py:class}`~numpy.longdouble` does **not** have quadruple precision, a nonstandard quadruple-precision type is instead available in C/C++ (this is the case, for instance, on x86-64 Linux and on some PowerPC platforms). On such platforms, and if the heyoka C++ library was compiled with support for the [mp++ library](https://github.com/bluescarni/mppp), quadruple precision is supported via the ``real128`` type (as shown above).\n", "\n", "Note that the non-IEEE {py:class}`~numpy.longdouble` type available on some PowerPC platforms (which implements a double-length floating-point representation with 106 significant bits) is **not** supported by heyoka.py at this time.\n", @@ -184,7 +184,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/doc/notebooks/ext_precision.ipynb b/doc/notebooks/ext_precision.ipynb index 119adb09..e20d2862 100644 --- a/doc/notebooks/ext_precision.ipynb +++ b/doc/notebooks/ext_precision.ipynb @@ -19,7 +19,7 @@ "\n", "- if you are using an Intel x86 processor and your C/C++ compiler implements ``long double`` as the 80-bit IEEE extended-precision format, then the 80-bit IEEE floating-point type is available as the NumPy {py:class}`~numpy.longdouble` type;\n", "- if you are on a platform with support for the non-standard ``__float128`` type, then the 128-bit IEEE floating-point type is available as the ``heyoka.real128`` type;\n", - "- if you are on a platform where the C/C++ ``long double`` type is implemented as a quadruple-precision IEEE floating-point type (e.g., 64-bit ARM), then the 128-bit IEEE floating-point type is available as the NumPy {py:class}`~numpy.longdouble` type.\n", + "- if you are on a platform where the C/C++ ``long double`` type is implemented as a quadruple-precision IEEE floating-point type (e.g., 64-bit Linux ARM), then the 128-bit IEEE floating-point type is available as the NumPy {py:class}`~numpy.longdouble` type.\n", "\n", "In other words, extended precision computations in heyoka.py are supported via the NumPy {py:class}`~numpy.longdouble` type (which could be either an 80-bit or 128-bit floating-point type, depending on the platform) and the ``heyoka.real128`` type (which will always be a 128-bit floating-point type, if available).\n", "\n", @@ -415,7 +415,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.12.4" } }, "nbformat": 4, From 148245864e395c1c47c8973bd1ed4e4283aedc05 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 13:15:50 +0200 Subject: [PATCH 09/25] Another notebook section fix. --- doc/notebooks/differentiable_atmosphere.ipynb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/notebooks/differentiable_atmosphere.ipynb b/doc/notebooks/differentiable_atmosphere.ipynb index aed421ca..42fd8804 100644 --- a/doc/notebooks/differentiable_atmosphere.ipynb +++ b/doc/notebooks/differentiable_atmosphere.ipynb @@ -17,7 +17,7 @@ "*Izzo, Dario*, *Giacomo Acciarini*, and *Francesco Biscani*. \"NeuralODEs for VLEO simulations: Introducing thermoNET for Thermosphere Modeling.\" arXiv preprint arXiv:2405.19384 (2024).\n", "```\n", "\n", - "# Background\n", + "## Background\n", "\n", "We leverage the key observation that the thermosphere is well approximated (at any given time, and location) by an exponential behavior with respect to the altitude. \n", "\n", @@ -791,7 +791,7 @@ ], "metadata": { "kernelspec": { - "display_name": "diff_drag_py38", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -805,10 +805,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" - }, - "orig_nbformat": 4 + "version": "3.12.4" + } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } From aae272a9a906d1ff72a35f0c171c7f40a9dfba49 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 13:17:53 +0200 Subject: [PATCH 10/25] Revert "Tentative windows debugging." This reverts commit 801170a4001feb0cd731298795dd34a82f612e81. --- heyoka/core.cpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/heyoka/core.cpp b/heyoka/core.cpp index 6a9b16f1..58085e17 100644 --- a/heyoka/core.cpp +++ b/heyoka/core.cpp @@ -396,24 +396,6 @@ PYBIND11_MODULE(core, m) #endif heypy::detail::tbb_gc.reset(); })); - - std::set_terminate([]() { - try { - std::exception_ptr eptr{std::current_exception()}; - if (eptr) { - std::rethrow_exception(eptr); - } else { - std::cerr << "Exiting without exception\n"; - } - } catch (const std::exception &ex) { - std::cerr << "Exception: " << ex.what() << std::endl; - std::cout << "Exception: " << ex.what() << std::endl; - } catch (...) { - std::cerr << "Unknown exception caught" << std::endl; - std::cout << "Unknown exception caught" << std::endl; - } - std::exit(EXIT_FAILURE); - }); } #if defined(__clang__) From 8a0aaf9018ce85b719ebdd0c2b2205e87865dd66 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 13:23:52 +0200 Subject: [PATCH 11/25] Try switching around the test order. --- heyoka/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/heyoka/test.py b/heyoka/test.py index a7615143..da54fbc4 100644 --- a/heyoka/test.py +++ b/heyoka/test.py @@ -2095,9 +2095,7 @@ def run_test_suite(): tl = _ut.TestLoader() - suite = tl.loadTestsFromTestCase( - _test_scalar_integrator.scalar_integrator_test_case - ) + suite = tl.loadTestsFromTestCase(_test_model.model_test_case) suite.addTest( tl.loadTestsFromTestCase(_test_batch_integrator.batch_integrator_test_case) ) @@ -2113,7 +2111,9 @@ def run_test_suite(): suite.addTest(tl.loadTestsFromTestCase(_test_elp2000.elp2000_test_case)) suite.addTest(tl.loadTestsFromTestCase(_test_dtens.dtens_test_case)) suite.addTest(tl.loadTestsFromTestCase(_test_mp.mp_test_case)) - suite.addTest(tl.loadTestsFromTestCase(_test_model.model_test_case)) + suite.addTest( + tl.loadTestsFromTestCase(_test_scalar_integrator.scalar_integrator_test_case) + ) suite.addTest(tl.loadTestsFromTestCase(_test_real.real_test_case)) suite.addTest(tl.loadTestsFromTestCase(_test_real128.real128_test_case)) suite.addTest(tl.loadTestsFromTestCase(_test_cfunc.cfunc_test_case)) From 53d5611dc8c392205cfa4127b0485d252a7be321 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 13:53:27 +0200 Subject: [PATCH 12/25] Fix numpy logical ops usage in the sgp4 prop tutorial, add suggestion about compact mode. --- doc/notebooks/sgp4_propagator.ipynb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/notebooks/sgp4_propagator.ipynb b/doc/notebooks/sgp4_propagator.ipynb index cac0c5e0..1f9d842d 100644 --- a/doc/notebooks/sgp4_propagator.ipynb +++ b/doc/notebooks/sgp4_propagator.ipynb @@ -425,11 +425,11 @@ "# - generated an error code, or\n", "# - ended up farther than 8000km from the Earth, or\n", "# - contain non-finite positional data.\n", - "mask = np.logical_and(\n", + "mask = np.logical_and.reduce((\n", " e[:, 0] == 0,\n", " np.linalg.norm(r[:, 0], axis=1) < 8000,\n", - " np.all(np.isfinite(r[:, 0]), axis=1),\n", - ")" + " np.all(np.isfinite(r[:, 0]), axis=1)\n", + "))" ] }, { @@ -658,11 +658,11 @@ "fig = plt.figure(figsize=(8,4))\n", "\n", "for i, ssv in enumerate(batch_sv):\n", - " cur_mask = np.logical_and(\n", + " cur_mask = np.logical_and.reduce((\n", " batch_e[:, i] == 0,\n", " np.linalg.norm(batch_r[:, i], axis=1) < 8000,\n", " np.all(np.isfinite(batch_r[:, i]), axis=1),\n", - " )\n", + " ))\n", " \n", " # Compute the positional error in meters\n", " # for the current date.\n", @@ -708,7 +708,7 @@ "outputs": [], "source": [ "# Build a first-order differentiable propagator.\n", - "dprop = hy.model.sgp4_propagator(sats, diff_order=1)" + "dprop = hy.model.sgp4_propagator(sats, diff_order=1, compact_mode=True)" ] }, { @@ -716,6 +716,8 @@ "id": "1c89b650-3ade-4b62-8924-626b5b4de7c4", "metadata": {}, "source": [ + "Note how we enabled the ``compact_mode`` flag: the computation of the derivatives will generate very large symbolic expressions, and enabling compact mode allows us to keep the compilation times at a manageable level.\n", + "\n", "We can confirm that ``dprop`` is a differentiable propagator by taking a look at its {py:attr}`~heyoka.model.sgp4_propagator_dbl.diff_order` property:" ] }, From e04b794f007ffbf40b39acf31c4a726e9a691219 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 14:04:53 +0200 Subject: [PATCH 13/25] Make a debug attempt without numba. --- .github/workflows/gh_actions_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh_actions_ci.yml b/.github/workflows/gh_actions_ci.yml index a50ec397..4d44d581 100644 --- a/.github/workflows/gh_actions_ci.yml +++ b/.github/workflows/gh_actions_ci.yml @@ -140,7 +140,7 @@ jobs: cmake --build . --config Release --target install -j2 cd .. cd tools - python ci_test_runner.py --with-numba + python ci_test_runner.py binder_cache: runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' From 3741f6e651e375c17bc2c0672855d3526957d85e Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 14:22:03 +0200 Subject: [PATCH 14/25] Another try at updating the action versions for the manylinux builds. --- .github/workflows/gh_actions_ci.yml | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/gh_actions_ci.yml b/.github/workflows/gh_actions_ci.yml index 4d44d581..89ad47d3 100644 --- a/.github/workflows/gh_actions_ci.yml +++ b/.github/workflows/gh_actions_ci.yml @@ -41,13 +41,10 @@ jobs: HEYOKA_PY_BUILD_TYPE: "Python312" TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} steps: - # NOTE: don't updated to checkout@v4 here because - # the action does not work properly due to missing - # symbols on the ancient manylinux images. - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build run: bash tools/gha_manylinux.sh - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: wheel_312 path: build/wheel/dist2/*.whl @@ -59,10 +56,10 @@ jobs: HEYOKA_PY_BUILD_TYPE: "Python311" TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build run: bash tools/gha_manylinux.sh - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: wheel_311 path: build/wheel/dist2/*.whl @@ -74,10 +71,10 @@ jobs: HEYOKA_PY_BUILD_TYPE: "Python310" TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build run: bash tools/gha_manylinux.sh - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: wheel_310 path: build/wheel/dist2/*.whl @@ -89,10 +86,10 @@ jobs: HEYOKA_PY_BUILD_TYPE: "Python39" TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build run: bash tools/gha_manylinux.sh - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: wheel_39 path: build/wheel/dist2/*.whl @@ -104,10 +101,10 @@ jobs: HEYOKA_PY_BUILD_TYPE: "Python38" TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build run: bash tools/gha_manylinux.sh - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: wheel_38 path: build/wheel/dist2/*.whl From a4903d6d042810e2d4a30467fc6f7f86bbe23408 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 14:22:25 +0200 Subject: [PATCH 15/25] Some testing for the new parjit and code_model options. --- heyoka/_test_batch_integrator.py | 22 +++++++++++++++++++-- heyoka/_test_cfunc.py | 32 ++++++++++++++++++++++++++----- heyoka/_test_scalar_integrator.py | 23 ++++++++++++++++++++-- heyoka/_test_sgp4_propagator.py | 12 +++++++++--- 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/heyoka/_test_batch_integrator.py b/heyoka/_test_batch_integrator.py index bc6c9c20..ae031ee1 100644 --- a/heyoka/_test_batch_integrator.py +++ b/heyoka/_test_batch_integrator.py @@ -15,20 +15,38 @@ def test_llvm_state_settings(self): # are correctly propagated through the integrator # constructor. - from . import taylor_adaptive_batch + from . import taylor_adaptive_batch, code_model from .model import pendulum + from sys import getrefcount ta = taylor_adaptive_batch(pendulum(), [[0.0, 0.0], [0.0, 0.0]]) + # Check correct reference count handling of the + # llvm_state property. + rc = getrefcount(ta) + tmp = ta.llvm_state + self.assertEqual(getrefcount(ta), rc + 1) + self.assertFalse(ta.llvm_state.force_avx512) self.assertFalse(ta.llvm_state.slp_vectorize) ta = taylor_adaptive_batch( - pendulum(), [[0.0, 0.0], [0.0, 0.0]], force_avx512=True, slp_vectorize=True + pendulum(), + [[0.0, 0.0], [0.0, 0.0]], + force_avx512=True, + slp_vectorize=True, + parjit=True, + compact_mode=True, + code_model=code_model.large, ) + rc = getrefcount(ta) + tmp = ta.llvm_state + self.assertEqual(getrefcount(ta), rc + 1) + self.assertTrue(ta.llvm_state.force_avx512) self.assertTrue(ta.llvm_state.slp_vectorize) + self.assertEqual(ta.llvm_state.code_model, code_model.large) def test_type_conversions(self): # Test to check automatic conversions of std::vector diff --git a/heyoka/_test_cfunc.py b/heyoka/_test_cfunc.py index 191a5ce4..3ad7cfbd 100644 --- a/heyoka/_test_cfunc.py +++ b/heyoka/_test_cfunc.py @@ -12,15 +12,22 @@ class cfunc_test_case(_ut.TestCase): def test_basic(self): - from . import cfunc, make_vars, cfunc_dbl, core, par, time + from . import cfunc, make_vars, cfunc_dbl, core, par, time, code_model import pickle from copy import copy, deepcopy + from sys import getrefcount self.assertRaises(ValueError, lambda: cfunc([], [])) x, y, z, s = make_vars("x", "y", "z", "s") cf = cfunc([y * (x + z)], [x, y, z]) + # Ensure that when extracting the llvm states + # the ref count of cf is properly updated. + rc = getrefcount(cf) + tmp = cf.llvm_states + self.assertEqual(getrefcount(cf), rc + 3) + self.assertTrue(all(not _.force_avx512 for _ in cf.llvm_states)) self.assertTrue(all(not _.slp_vectorize for _ in cf.llvm_states)) @@ -47,11 +54,26 @@ def test_basic(self): [_.get_ir() for _ in cf.llvm_states], ) - cf = cfunc([y * (x + z)], vars=[y, z, x], force_avx512=True, slp_vectorize=True) - self.assertEqual(cf.vars, [y, z, x]) + # Construct with custom LLVM settings too. + cf = cfunc( + [y * (x + z)], + vars=[y, z, x], + force_avx512=True, + slp_vectorize=True, + compact_mode=True, + parjit=False, + code_model=code_model.large, + ) - self.assertTrue(all(_.force_avx512 for _ in cf.llvm_states)) - self.assertTrue(all(_.slp_vectorize for _ in cf.llvm_states)) + rc = getrefcount(cf) + tmp = cf.llvm_states + self.assertEqual(getrefcount(cf), rc + 1) + + self.assertEqual(cf.vars, [y, z, x]) + self.assertTrue(cf.llvm_states.force_avx512) + self.assertTrue(cf.llvm_states.slp_vectorize) + self.assertFalse(cf.llvm_states.parjit) + self.assertEqual(cf.llvm_states.code_model, code_model.large) # Tests for correct detection of number of params, time dependency # and list of variables. diff --git a/heyoka/_test_scalar_integrator.py b/heyoka/_test_scalar_integrator.py index 88b677a6..72a60e47 100644 --- a/heyoka/_test_scalar_integrator.py +++ b/heyoka/_test_scalar_integrator.py @@ -15,20 +15,39 @@ def test_llvm_state_settings(self): # are correctly propagated through the integrator # constructor. - from . import taylor_adaptive + from . import taylor_adaptive, code_model from .model import pendulum + from sys import getrefcount ta = taylor_adaptive(pendulum(), [0.0, 0.0]) + # Check correct reference count handling of the + # llvm_state property. + rc = getrefcount(ta) + tmp = ta.llvm_state + self.assertEqual(getrefcount(ta), rc + 1) + self.assertFalse(ta.llvm_state.force_avx512) self.assertFalse(ta.llvm_state.slp_vectorize) ta = taylor_adaptive( - pendulum(), [0.0, 0.0], force_avx512=True, slp_vectorize=True + pendulum(), + [0.0, 0.0], + force_avx512=True, + slp_vectorize=True, + parjit=True, + compact_mode=True, + code_model=code_model.large, ) + rc = getrefcount(ta) + tmp = ta.llvm_state + self.assertEqual(getrefcount(ta), rc + 1) + self.assertTrue(ta.llvm_state.force_avx512) self.assertTrue(ta.llvm_state.slp_vectorize) + self.assertTrue(ta.llvm_state.parjit) + self.assertEqual(ta.llvm_state.code_model, code_model.large) def test_type_conversions(self): # Test to check automatic conversions of std::vector diff --git a/heyoka/_test_sgp4_propagator.py b/heyoka/_test_sgp4_propagator.py index c48ff0d7..71a612e5 100644 --- a/heyoka/_test_sgp4_propagator.py +++ b/heyoka/_test_sgp4_propagator.py @@ -26,7 +26,7 @@ def test_basics(self): from .model import sgp4_propagator from pickle import loads, dumps from copy import copy, deepcopy - from . import make_vars + from . import make_vars, code_model s1 = sgp4_propagator_test_case.s1 t1 = sgp4_propagator_test_case.t1 @@ -106,8 +106,14 @@ def test_basics(self): self.assertEqual(cprop.sat_data[6, 0], 0.14841e-3) self.assertEqual(cprop.sat_data[6, 1], 0.46268e-3) - # A couple of tests with the derivatives. - prop = sgp4_propagator([sat1, sat2], diff_order=1) + # A couple of tests with the derivatives and custom llvm settings. + prop = sgp4_propagator( + [sat1, sat2], + diff_order=1, + parjit=False, + compact_mode=True, + code_model=code_model.large, + ) self.assertEqual( prop.diff_args, make_vars("n0", "e0", "i0", "node0", "omega0", "m0", "bstar"), From e189c995666eefe125f3b49d13f831cf79886e02 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 14:24:48 +0200 Subject: [PATCH 16/25] Update the circleci image version. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 992e7e2b..2927a2ef 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ jobs: command: bash ./tools/circleci_conda_heyoka_head_312.sh ubuntu_arm64: machine: - image: ubuntu-2004:202101-01 + image: ubuntu-2204:current resource_class: arm.large steps: - checkout From e56ffe949becd4c849dbc78b9cd49298b79f716f Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sat, 31 Aug 2024 14:25:29 +0200 Subject: [PATCH 17/25] Revert "Make a debug attempt without numba." This reverts commit e04b794f007ffbf40b39acf31c4a726e9a691219. --- .github/workflows/gh_actions_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh_actions_ci.yml b/.github/workflows/gh_actions_ci.yml index 89ad47d3..35cc681f 100644 --- a/.github/workflows/gh_actions_ci.yml +++ b/.github/workflows/gh_actions_ci.yml @@ -137,7 +137,7 @@ jobs: cmake --build . --config Release --target install -j2 cd .. cd tools - python ci_test_runner.py + python ci_test_runner.py --with-numba binder_cache: runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' From 9a891a499855773c680a2f556a53a96032327165 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Sun, 1 Sep 2024 09:47:21 +0200 Subject: [PATCH 18/25] Ping for CI. From 908844be8d6ac603d5ff730041689d75da65bf4a Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Mon, 2 Sep 2024 14:49:32 +0200 Subject: [PATCH 19/25] Make an attempt with clang-cl on Windows. --- .github/workflows/gh_actions_ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gh_actions_ci.yml b/.github/workflows/gh_actions_ci.yml index 35cc681f..800927df 100644 --- a/.github/workflows/gh_actions_ci.yml +++ b/.github/workflows/gh_actions_ci.yml @@ -122,19 +122,19 @@ jobs: - name: Build shell: pwsh run: | - conda install -y python=3.12 git pybind11 numpy<2 cmake llvmdev tbb-devel tbb astroquery libboost-devel fmt<11 skyfield spdlog sleef sympy cloudpickle zlib libzlib 'mppp=1.*' numba + conda install -y clang ninja python=3.12 git pybind11 numpy<2 cmake llvmdev tbb-devel tbb astroquery libboost-devel fmt skyfield spdlog sleef sympy cloudpickle zlib libzlib 'mppp=1.*' numba git clone --depth 1 https://github.com/bluescarni/heyoka.git heyoka_cpp cd heyoka_cpp mkdir build cd build - cmake .. -G "Visual Studio 17 2022" -A x64 -DCMAKE_PREFIX_PATH=C:\Miniconda\envs\test\Library -DCMAKE_INSTALL_PREFIX=C:\Miniconda\envs\test\Library -DBoost_NO_BOOST_CMAKE=ON -DHEYOKA_ENABLE_IPO=yes -DHEYOKA_WITH_SLEEF=yes -DHEYOKA_WITH_MPPP=yes - cmake --build . --config Release --target install -j2 + cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_PREFIX_PATH=C:\Miniconda\envs\test\Library -DCMAKE_INSTALL_PREFIX=C:\Miniconda\envs\test\Library -DHEYOKA_ENABLE_IPO=yes -DHEYOKA_WITH_SLEEF=yes -DHEYOKA_WITH_MPPP=yes + cmake --build . --target install -j4 cd .. cd .. mkdir build cd build - cmake .. -G "Visual Studio 17 2022" -A x64 -DCMAKE_PREFIX_PATH=C:\Miniconda\envs\test\Library -DCMAKE_INSTALL_PREFIX=C:\Miniconda\envs\test\Library -DBoost_NO_BOOST_CMAKE=ON -DHEYOKA_PY_ENABLE_IPO=yes - cmake --build . --config Release --target install -j2 + cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_PREFIX_PATH=C:\Miniconda\envs\test\Library -DCMAKE_INSTALL_PREFIX=C:\Miniconda\envs\test\Library -DHEYOKA_PY_ENABLE_IPO=yes + cmake --build . --target install -j4 cd .. cd tools python ci_test_runner.py --with-numba From 59b5fe8715697488e173e7909021496b4feccfd6 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Mon, 2 Sep 2024 15:14:52 +0200 Subject: [PATCH 20/25] Try downgrading boost. --- .github/workflows/gh_actions_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh_actions_ci.yml b/.github/workflows/gh_actions_ci.yml index 800927df..5910e8cc 100644 --- a/.github/workflows/gh_actions_ci.yml +++ b/.github/workflows/gh_actions_ci.yml @@ -122,7 +122,7 @@ jobs: - name: Build shell: pwsh run: | - conda install -y clang ninja python=3.12 git pybind11 numpy<2 cmake llvmdev tbb-devel tbb astroquery libboost-devel fmt skyfield spdlog sleef sympy cloudpickle zlib libzlib 'mppp=1.*' numba + conda install -y clang ninja python=3.12 git pybind11 numpy<2 cmake llvmdev tbb-devel tbb astroquery libboost-devel<=1.84 fmt skyfield spdlog sleef sympy cloudpickle zlib libzlib 'mppp=1.*' numba git clone --depth 1 https://github.com/bluescarni/heyoka.git heyoka_cpp cd heyoka_cpp mkdir build From 64f3284c37a8b095a8cab65063d4f724e1d39ff2 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Wed, 4 Sep 2024 10:02:22 +0200 Subject: [PATCH 21/25] Fix warning in the build system when locating Boost. --- CMakeLists.txt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6dbb7525..f4411167 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,8 +166,19 @@ if(${pybind11_VERSION} VERSION_LESS "2.10") endif() # Boost. +# NOTE: we look for Boost in CONFIG mode first, as that has become the official supported way +# of locating Boost in recent Boost/CMake versions. If we fail, we try again in +# MODULE mode as last resort. # NOTE: need 1.69 for safe numerics. -find_package(Boost 1.69 REQUIRED serialization) +find_package(Boost 1.69 QUIET COMPONENTS serialization CONFIG) +if(NOT ${Boost_FOUND}) + message(STATUS "Boost not found in CONFIG mode, retrying in MODULE mode.") + find_package(Boost 1.69 QUIET MODULE COMPONENTS serialization) +endif() +if(NOT ${Boost_FOUND}) + message(FATAL_ERROR "Could not locate Boost in either CONFIG or MODULE mode.") +endif() +message(STATUS "Found Boost version ${Boost_VERSION}.") # Mandatory dependency on TBB. find_package(TBB REQUIRED CONFIG) From 9c3e78a2768756068a353cdfa5c03942ac02bfeb Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Wed, 4 Sep 2024 10:04:58 +0200 Subject: [PATCH 22/25] Go back to MSVC in the Windows CI. --- .github/workflows/gh_actions_ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gh_actions_ci.yml b/.github/workflows/gh_actions_ci.yml index 5910e8cc..ec4e111d 100644 --- a/.github/workflows/gh_actions_ci.yml +++ b/.github/workflows/gh_actions_ci.yml @@ -122,18 +122,18 @@ jobs: - name: Build shell: pwsh run: | - conda install -y clang ninja python=3.12 git pybind11 numpy<2 cmake llvmdev tbb-devel tbb astroquery libboost-devel<=1.84 fmt skyfield spdlog sleef sympy cloudpickle zlib libzlib 'mppp=1.*' numba + conda install -y python=3.12 git pybind11 numpy<2 cmake llvmdev tbb-devel tbb astroquery libboost-devel fmt skyfield spdlog sleef sympy cloudpickle zlib libzlib 'mppp=1.*' numba git clone --depth 1 https://github.com/bluescarni/heyoka.git heyoka_cpp cd heyoka_cpp mkdir build cd build - cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_PREFIX_PATH=C:\Miniconda\envs\test\Library -DCMAKE_INSTALL_PREFIX=C:\Miniconda\envs\test\Library -DHEYOKA_ENABLE_IPO=yes -DHEYOKA_WITH_SLEEF=yes -DHEYOKA_WITH_MPPP=yes + cmake .. -G "Visual Studio 17 2022" -A x64 -DCMAKE_PREFIX_PATH=C:\Miniconda\envs\test\Library -DCMAKE_INSTALL_PREFIX=C:\Miniconda\envs\test\Library -DHEYOKA_ENABLE_IPO=yes -DHEYOKA_WITH_SLEEF=yes -DHEYOKA_WITH_MPPP=yes cmake --build . --target install -j4 cd .. cd .. mkdir build cd build - cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_PREFIX_PATH=C:\Miniconda\envs\test\Library -DCMAKE_INSTALL_PREFIX=C:\Miniconda\envs\test\Library -DHEYOKA_PY_ENABLE_IPO=yes + cmake .. -G "Visual Studio 17 2022" -A x64 -DCMAKE_PREFIX_PATH=C:\Miniconda\envs\test\Library -DCMAKE_INSTALL_PREFIX=C:\Miniconda\envs\test\Library -DHEYOKA_PY_ENABLE_IPO=yes cmake --build . --target install -j4 cd .. cd tools From 65bc69eb31012cc594efc17b83ae5f177eb2042d Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Wed, 4 Sep 2024 10:24:46 +0200 Subject: [PATCH 23/25] Add warnings about the use of incompatible compilers when building from source. --- doc/install.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/install.rst b/doc/install.rst index 979b2d3f..eb37ae35 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -21,8 +21,7 @@ heyoka.py has several Python and C++ dependencies. On the C++ side, heyoka.py de :ref:`heyoka ` and :ref:`mp++ ` installation instructions). -On the Python side, heyoka.py requires at least Python 3.5 -(Python 2.x is **not** supported) and depends on: +On the Python side, heyoka.py requires at least Python 3.5 and depends on: * `NumPy `__ (version <2, **mandatory**), * `cloudpickle `__ (**mandatory**), @@ -88,6 +87,13 @@ heyoka.py. In order to install heyoka.py using pkg, execute the following comman Installation from source ------------------------ +.. warning:: + + When installing from source, it is important to ensure that the compiler that was used + to build heyoka.py's dependencies is ABI-compatible with the compiler that is being used + to build heyoka.py. Mixing incompatible compilers will likely lead to hard-to-diagnose + build and/or runtime issues. + heyoka.py is written in modern C++, and it requires a compiler able to understand at least C++20. The library is regularly tested on a continuous integration pipeline which currently includes several From 74cb4e09a12eef56e0218f32d973288769c28060 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Wed, 4 Sep 2024 10:46:46 +0200 Subject: [PATCH 24/25] Fix building on Windows CI. --- .github/workflows/gh_actions_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh_actions_ci.yml b/.github/workflows/gh_actions_ci.yml index ec4e111d..2da60aa6 100644 --- a/.github/workflows/gh_actions_ci.yml +++ b/.github/workflows/gh_actions_ci.yml @@ -128,13 +128,13 @@ jobs: mkdir build cd build cmake .. -G "Visual Studio 17 2022" -A x64 -DCMAKE_PREFIX_PATH=C:\Miniconda\envs\test\Library -DCMAKE_INSTALL_PREFIX=C:\Miniconda\envs\test\Library -DHEYOKA_ENABLE_IPO=yes -DHEYOKA_WITH_SLEEF=yes -DHEYOKA_WITH_MPPP=yes - cmake --build . --target install -j4 + cmake --build . --config Release --target install -j4 cd .. cd .. mkdir build cd build cmake .. -G "Visual Studio 17 2022" -A x64 -DCMAKE_PREFIX_PATH=C:\Miniconda\envs\test\Library -DCMAKE_INSTALL_PREFIX=C:\Miniconda\envs\test\Library -DHEYOKA_PY_ENABLE_IPO=yes - cmake --build . --target install -j4 + cmake --build . --config Release --target install -j4 cd .. cd tools python ci_test_runner.py --with-numba From 08c15496ea844d039fbce401447869d58f5d7e34 Mon Sep 17 00:00:00 2001 From: Francesco Biscani Date: Wed, 4 Sep 2024 12:05:46 +0200 Subject: [PATCH 25/25] Update changelog. --- doc/changelog.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index e5370911..e3be509a 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -3,6 +3,33 @@ Changelog ========= +6.0.0 (unreleased) +------------------ + +New +~~~ + +- Implement parallel compilation for Taylor integrators + and compiled functions + (`#188 `__). +- Add the possibility of specifying the LLVM code model + used for JIT compilation + (`#188 `__). + +Changes +~~~~~~~ + +- heyoka.py now requires version 6.0.0 of the + heyoka C++ library + (`#188 `__). + +Fix +~~~ + +- Fix build system warnings when using recent versions of + CMake and Boost + (`#188 `__). + 5.1.0 (2024-07-23) ------------------