From 9215b76d6c2470e6b70b1102eb00107dec805a13 Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Tue, 7 Jul 2020 17:42:22 -0400 Subject: [PATCH 1/2] DOC: more tutorial content --- docs/source/appendix.rst | 72 +++--- docs/source/conf.py | 1 + docs/source/index.rst | 1 + docs/source/shock-and-awe.rst | 198 ++++++++++++++++ docs/source/tutorial.rst | 221 +----------------- docs/source/tutorial/arrays.rst | 133 +++++++++++ docs/source/tutorial/functions.rst | 105 +++++++++ docs/source/tutorial/libpy_tutorial/arrays.cc | 10 +- .../tutorial/libpy_tutorial/function.cc | 15 ++ docs/source/tutorial/setup.py | 4 + include/libpy/automodule.h | 18 +- libpy/build.py | 6 +- 12 files changed, 529 insertions(+), 255 deletions(-) create mode 100644 docs/source/shock-and-awe.rst create mode 100644 docs/source/tutorial/arrays.rst create mode 100644 docs/source/tutorial/functions.rst create mode 100644 docs/source/tutorial/libpy_tutorial/function.cc diff --git a/docs/source/appendix.rst b/docs/source/appendix.rst index e999c28f..b9f3fbfd 100644 --- a/docs/source/appendix.rst +++ b/docs/source/appendix.rst @@ -2,15 +2,11 @@ Appendix ======== -Python Object References -======================== - -.. doxygenclass:: py::owned_ref - :members: - :undoc-members: +C++ +=== ```` -================= +----------------- .. doxygenstruct:: py::abi::abi_version :undoc-members: @@ -22,7 +18,7 @@ Python Object References .. doxygenfunction:: py::abi::ensure_compatible_libpy_abi ```` -================= +----------------- .. doxygenclass:: py::any_vtable :members: @@ -43,14 +39,14 @@ Python Object References .. doxygenfunction:: py::dtype_to_vtable ```` -======================== +------------------------ .. doxygenclass:: py::any_vector :members: :undoc-members: ```` -======================= +----------------------- .. doxygenstruct:: py::autoclass :members: @@ -62,7 +58,7 @@ Python Object References :members: ```` -========================== +-------------------------- .. doxygenfunction:: py::autofunction @@ -86,15 +82,21 @@ Python Object References .. doxygenclass:: py::dispatch::adapt_argument :members: + +```` +------------------------ + +.. doxygendefine:: LIBPY_AUTOMODULE + ```` -========================== +-------------------------- .. doxygenclass:: py::borrowed_ref :members: :undoc-members: ```` -==================== +-------------------- .. doxygentypedef:: py::buffer @@ -107,12 +109,12 @@ Python Object References .. doxygenfunction:: py::buffer_type_compatible(const py::buffer&) ```` -========================= +------------------------- .. doxygenfunction:: py::build_tuple ```` -=========================== +--------------------------- .. doxygenfunction:: py::call_function @@ -123,7 +125,7 @@ Python Object References .. doxygenfunction:: py::call_method_throws ```` -=========================== +--------------------------- .. doxygentypedef:: py::cs::char_sequence @@ -144,7 +146,7 @@ Python Object References .. doxygenfunction:: py::cs::join ```` -======================== +------------------------ .. doxygenclass:: py::datetime64 :members: @@ -165,7 +167,7 @@ Python Object References .. doxygenfunction:: py::chrono::time_since_epoch ```` -====================== +---------------------- .. doxygenfunction:: py::util::demangle_string(const char*) @@ -176,14 +178,14 @@ Python Object References .. doxygenclass:: py::util::demangle_error ```` -======================== +------------------------ .. doxygenclass:: py::dict_range :members: :undoc-members: ```` -======================= +----------------------- .. doxygenclass:: py::exception :members: @@ -195,7 +197,7 @@ Python Object References .. doxygenstruct:: py::dispatch::raise_format ```` -========================= +------------------------- .. doxygenfunction:: py::from_object @@ -204,7 +206,7 @@ Python Object References .. doxygenstruct:: py::dispatch::from_object ```` -===================== +--------------------- .. doxygenfunction:: py::getattr @@ -215,13 +217,13 @@ Python Object References .. doxygenfunction:: py::nested_getattr_throws ```` -================= +----------------- .. doxygenstruct:: py::gil :members: ```` -================== +------------------ .. doxygenfunction:: py::hash_combine(T, Ts...) @@ -232,7 +234,7 @@ Python Object References .. doxygenfunction:: py::hash_buffer ```` -======================= +----------------------- .. doxygenfunction:: py::zip @@ -241,7 +243,7 @@ Python Object References .. doxygenfunction:: py::imap ```` -================== +------------------ .. doxygenstruct:: py::meta::print_t @@ -258,7 +260,7 @@ Python Object References .. doxygentypedef:: py::meta::set_diff ``op`` operator function objects --------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Each of these types implements ``operator()`` to defer to the named operator while attempting to preserve all the observable properties of calling the underlying operator directly. @@ -284,7 +286,7 @@ Each of these types implements ``operator()`` to defer to the named operator whi .. doxygenstruct:: py::meta::op::inv ```` -========================== +-------------------------- .. doxygenclass:: py::ndarray_view :members: @@ -318,3 +320,17 @@ These partial specializations implement the same protocol as the non type-erased .. doxygenclass:: py::ndarray_view< any_ref, 1, false > :members: :undoc-members: + +Python +====== + +Miscellaneous +------------- +.. autodata:: libpy.version_info + + The ABI version of libpy. + +Build +----- + +.. autoclass:: libpy.build.LibpyExtension diff --git a/docs/source/conf.py b/docs/source/conf.py index 35a4f931..3ff6edaa 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -9,6 +9,7 @@ 'breathe', 'IPython.sphinxext.ipython_console_highlighting', 'IPython.sphinxext.ipython_directive', + 'sphinx.ext.autodoc', ] breathe_projects = {'libpy': '../doxygen-build/xml'} diff --git a/docs/source/index.rst b/docs/source/index.rst index 778a1895..71f55cfc 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,7 @@ Welcome to libpy's documentation! :maxdepth: 2 :caption: Contents: + shock-and-awe tutorial install appendix diff --git a/docs/source/shock-and-awe.rst b/docs/source/shock-and-awe.rst new file mode 100644 index 00000000..00ce4f9e --- /dev/null +++ b/docs/source/shock-and-awe.rst @@ -0,0 +1,198 @@ +==================== +Shock and Awe [#f1]_ +==================== + +A *concise* overview of ``libpy``. For an introduction to extending Python with C or C++ please see `the Python documentation `_ or `Joe Jevnik's C Extension Tutorial `_. + +Simple Scalar Functions +======================= + +We start by building simple scalar functions in C++ which we can call from Python. + +.. ipython:: python + + from libpy_tutorial import scalar_functions + +A simple scalar function: + +.. literalinclude:: tutorial/libpy_tutorial/scalar_functions.cc + :lines: 11-13 + +.. ipython:: python + + scalar_functions.bool_scalar(False) + +A great way to use ``libpy`` is to write the code that needs to be fast in C++ and expose that code via Python. Let's estimate ``pi`` using a monte carlo simulation: + +.. literalinclude:: tutorial/libpy_tutorial/scalar_functions.cc + :lines: 15-30 + +.. ipython:: python + + scalar_functions.monte_carlo_pi(10000000) + +Of course, we can build C++ functions that support all the features of regular Python functions. + +``libpy`` supports optional args: + +.. literalinclude:: tutorial/libpy_tutorial/scalar_functions.cc + :lines: 34-36 + +.. ipython:: python + + scalar_functions.optional_arg(b"An argument was passed") + scalar_functions.optional_arg() + +and keyword/optional keyword arguments: + +.. literalinclude:: tutorial/libpy_tutorial/scalar_functions.cc + :lines: 38-44 + +.. ipython:: python + + scalar_functions.keyword_args(kw_arg_kwd=1) + scalar_functions.keyword_args(kw_arg_kwd=1, opt_kw_arg_kwd=55) + + +Working With Arrays +=================== + +In order to write performant code it is often useful to write vectorized functions that act on arrays. Thus, libpy has extenstive support for ``numpy`` arrays. + +.. ipython:: python + + from libpy_tutorial import arrays + import numpy as np + +We can take ``numpy`` arrays as input: + +.. literalinclude:: tutorial/libpy_tutorial/arrays.cc + :lines: 11-17 + +.. ipython:: python + + some_numbers = np.arange(20000) + arrays.simple_sum(some_numbers) + +and return them as output: + +.. literalinclude:: tutorial/libpy_tutorial/arrays.cc + :lines: 23-43 + +.. ipython:: python + + prime_mask = arrays.is_prime(some_numbers) + some_numbers[prime_mask][:100] + +.. note:: ``numpy`` arrays passed to C++ are `ranges `_. + +.. literalinclude:: tutorial/libpy_tutorial/arrays.cc + :lines: 19-21 + +.. ipython:: python + + arrays.simple_sum_iterator(some_numbers) + +N Dimensional Arrays +==================== + +We can also work with n-dimensional arrays. As a motivating example, let's sharpen an image. Specifically - we will sharpen: + +.. ipython:: python + + from PIL import Image + import matplotlib.pyplot as plt # to show the image in documenation + import numpy as np + import pkg_resources + img_file = pkg_resources.resource_stream("libpy_tutorial", "data/original.png") + img = Image.open(img_file) + @savefig original.png width=200px + plt.imshow(img) + +.. literalinclude:: tutorial/libpy_tutorial/ndarrays.cc + :lines: 10-55 + +.. ipython:: python + + pixels = np.array(img) + kernel = np.array([ + [0, -1, 0], + [-1, 5, -1], + [0, -1, 0] + ]) # already normalized + from libpy_tutorial import ndarrays + res = ndarrays.apply_kernel(pixels, kernel) + @savefig sharpened.png width=200px + plt.imshow(res) + + +.. note:: We are able to pass a shaped n-dimensional array as input and return one as output. + + +Creating Classes +================ + +``libpy`` also allows you to construct C++ classes and then easily expose them as if they are regular Python classes. + +.. ipython:: python + + from libpy_tutorial.classes import Vec3d + +C++ classes are able to emulate all the features of Python classes: + +.. literalinclude:: tutorial/libpy_tutorial/classes.cc + :lines: 9-67 + +.. literalinclude:: tutorial/libpy_tutorial/classes.cc + :lines: 93-106 + +.. ipython:: python + + Vec3d.__doc__ + v = Vec3d(1, 2, 3) + v + str(v) + v.x(), v.y(), v.z() + w = Vec3d(4, 5, 6); w + v + w + v * w + v.magnitude() + +Exceptions +========== + +Working with exceptions is also important. + +.. ipython:: python + + from libpy_tutorial import exceptions + +We can throw exceptions in C++ that will then be dealt with in Python. Two patterns: + +1. Throw your own exception: ``throw py::exception(type, msg...)``, maybe in response to an exception from a C-API function. +2. Throw a C++ exception directly. + +.. literalinclude:: tutorial/libpy_tutorial/exceptions.cc + :lines: 11-17 + +:: + + In [40]: exceptions.throw_value_error(4) + --------------------------------------------------------------------------- + ValueError Traceback (most recent call last) + in + ----> 1 exceptions.throw_value_error(4) + + ValueError: You passed 4 and this is the exception + + In [41]: exceptions.raise_from_cxx() + --------------------------------------------------------------------------- + RuntimeError Traceback (most recent call last) + in + ----> 1 exceptions.raise_from_cxx() + + RuntimeError: a C++ exception was raised: Supposedly a bad argument was used + +.. rubric:: Footnotes + +.. [#f1] With naming credit to the intorduction of `Q for Mortals `_. diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index b3e515ef..8ebbb51b 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -1,216 +1,9 @@ -==================== -Shock and Awe [#f1]_ -==================== +Tutorial +======== -A *concise* overview of ``libpy``. For an introduction to extending Python with C or C++ please see `the Python documentation `_ or `Joe Jevnik's C Extension Tutorial `_. +.. toctree:: + :maxdepth: 2 + :caption: Contents: -Simple Scalar Functions -======================= - -We start by building simple scalar functions in C++ which we can call from Python. - -.. ipython:: python - - from libpy_tutorial import scalar_functions - -A simple scalar function: - -.. literalinclude:: tutorial/libpy_tutorial/scalar_functions.cc - :lines: 11-13 - -.. ipython:: python - - scalar_functions.bool_scalar(False) - -A great way to use ``libpy`` is to write the code that needs to be fast in C++ and expose that code via Python. Let's estimate ``pi`` using a monte carlo simulation: - -.. literalinclude:: tutorial/libpy_tutorial/scalar_functions.cc - :lines: 15-30 - -.. ipython:: python - - scalar_functions.monte_carlo_pi(10000000) - -Of course, we can build C++ functions that support all the features of regular Python functions. - -``libpy`` supports optional args: - -.. literalinclude:: tutorial/libpy_tutorial/scalar_functions.cc - :lines: 34-36 - -.. ipython:: python - - scalar_functions.optional_arg(b"An argument was passed") - scalar_functions.optional_arg() - -and keyword/optional keyword arguments: - -.. literalinclude:: tutorial/libpy_tutorial/scalar_functions.cc - :lines: 38-44 - -.. ipython:: python - - scalar_functions.keyword_args(kw_arg_kwd=1) - scalar_functions.keyword_args(kw_arg_kwd=1, opt_kw_arg_kwd=55) - - -Working With Arrays -=================== - -In order to write performant code it is often useful to write vectorized functions that act on arrays. Thus, libpy has extenstive support for ``numpy`` arrays. - -.. ipython:: python - - from libpy_tutorial import arrays - import numpy as np - -We can take ``numpy`` arrays as input: - -.. literalinclude:: tutorial/libpy_tutorial/arrays.cc - :lines: 11-17 - -.. ipython:: python - - some_numbers = np.arange(20000) - arrays.simple_sum(some_numbers) - -and return them as output: - -.. literalinclude:: tutorial/libpy_tutorial/arrays.cc - :lines: 23-43 - -.. ipython:: python - - prime_mask = arrays.is_prime(some_numbers) - some_numbers[prime_mask][:100] - -.. note:: ``numpy`` arrays passed to C++ are `ranges `_. - -.. literalinclude:: tutorial/libpy_tutorial/arrays.cc - :lines: 19-21 - -.. ipython:: python - - arrays.simple_sum_iterator(some_numbers) - -N Dimensional Arrays -==================== - -We can also work with n-dimensional arrays. As a motivating example, let's sharpen an image. Specifically - we will sharpen: - -.. ipython:: python - - from PIL import Image - import matplotlib.pyplot as plt # to show the image in documenation - import numpy as np - import pkg_resources - img_file = pkg_resources.resource_stream("libpy_tutorial", "data/original.png") - img = Image.open(img_file) - @savefig original.png width=200px - plt.imshow(img) - -.. literalinclude:: tutorial/libpy_tutorial/ndarrays.cc - :lines: 10-55 - -.. ipython:: python - - pixels = np.array(img) - kernel = np.array([ - [0, -1, 0], - [-1, 5, -1], - [0, -1, 0] - ]) # already normalized - from libpy_tutorial import ndarrays - res = ndarrays.apply_kernel(pixels, kernel) - @savefig sharpened.png width=200px - plt.imshow(res) - - -.. note:: We are able to pass a shaped n-dimensional array as input and return one as output. - - -Creating Classes -================ - -``libpy`` also allows you to construct C++ classes and then easily expose them as if they are regular Python classes. - -.. ipython:: python - - from libpy_tutorial.classes import Vec3d - -C++ classes are able to emulate all the features of Python classes: - -.. literalinclude:: tutorial/libpy_tutorial/classes.cc - :lines: 9-67 - -.. literalinclude:: tutorial/libpy_tutorial/classes.cc - :lines: 93-106 - -.. ipython:: python - - Vec3d.__doc__ - v = Vec3d(1, 2, 3) - v - str(v) - v.x(), v.y(), v.z() - w = Vec3d(4, 5, 6); w - v + w - v * w - v.magnitude() - -Exceptions -========== - -Working with exceptions is also important. - -.. ipython:: python - - from libpy_tutorial import exceptions - -We can throw exceptions in C++ that will then be dealt with in Python. Two patterns: - -1. Throw your own exception: ``throw py::exception(type, msg...)``, maybe in response to an exception from a C-API function. -2. Throw a C++ exception directly. - -.. literalinclude:: tutorial/libpy_tutorial/exceptions.cc - :lines: 11-17 - -:: - - In [40]: exceptions.throw_value_error(4) - --------------------------------------------------------------------------- - ValueError Traceback (most recent call last) - in - ----> 1 exceptions.throw_value_error(4) - - ValueError: You passed 4 and this is the exception - - In [41]: exceptions.raise_from_cxx() - --------------------------------------------------------------------------- - RuntimeError Traceback (most recent call last) - in - ----> 1 exceptions.raise_from_cxx() - - RuntimeError: a C++ exception was raised: Supposedly a bad argument was used - -.. rubric:: Footnotes - -.. [#f1] With naming credit to the intorduction of `Q for Mortals `_. - -================= -Python Extensions -================= - -In order to create and use Python Extensions we must do two things: - -First, we use the ``LIBPY_AUTOMODULE`` macro to create and initialize the module: - -.. literalinclude:: tutorial/libpy_tutorial/scalar_functions.cc - :lines: 45-53 - -Next, we must tell ``setup.py`` to build our module using the ``LibpyExtension`` helper: - -.. literalinclude:: tutorial/setup.py - :lines: 18-28,37,50-54,71-72 - -Finally, we must ensure that we ``import libpy`` in Python before importing the extension module. + tutorial/functions + tutorial/arrays diff --git a/docs/source/tutorial/arrays.rst b/docs/source/tutorial/arrays.rst new file mode 100644 index 00000000..d565fa95 --- /dev/null +++ b/docs/source/tutorial/arrays.rst @@ -0,0 +1,133 @@ +====== +Arrays +====== + +To get the full power out of a C++ extension, you will often need to pass arrays of data between Python and C++. +Libpy has native support for integrating with numpy, the most popular ndarray library for Python. + +Libpy supports receiving arrays as views so that no data needs to be copied. +Libpy array views can also be const to guarantee that the underlying data isn't mutated. +Libpy also supports creating Numpy arrays as views over C++ containers without copying the underlying data. + +``py::array_view`` +================== + +Libpy can accept numpy arrays, or generally any buffer-like object, through a :cpp:struct:`py::ndarray_view`. +:cpp:struct:`py::ndarray_view` is a template type which takes as a parameter the C++ type of the elements of the array and the number of dimensions. +For example: ``py::ndarray_view`` is a view of a 3d array of signed 32 bit integers. +The type of the elements of a :cpp:struct:`py::ndarray_view` are fixed at compile time, but the shape is determined at runtime. + +As a convenience, :cpp:struct:`py::array_view` is an alias of :cpp:struct:`py::ndarray_view` for 1 dimensional arrays. + +Shape and Strides +----------------- + +Like numpy, an array view is composed of three parts: + +- shape :: ``std::array`` +- strides :: ``std::array`` +- buffer :: ``(const) std::byte*`` + +The shape array contains the number of elements along each axis. +For example: ``{2, 3}`` would be an array with 2 rows and 3 columns. + +The strides array contains the number of bytes needed to move one step along each axis. +For example: given a ``{2, 3}`` shaped array of 4 byte elements, then strides of ``{12, 4}`` would be a C-contiguous array because the rows are contiguous. +Given the same ``{2, 3}`` shaped array of 4 byte elements, then strides of ``{4, 8}`` would be a Fortran-contiguous array because the rows are contiguous. + +Non-contiguous views +-------------------- + +Array views do not need to view contiguous arrays. +For example, given a C-contiguous ``{4, 5}`` array of 2 byte values, we could take a view of first column by producing an array view with strides ``{10}``. + +Simple Array Input +================== + +Let's write function to sum an array: + +.. code-block:: c++ + + std::int64_t simple_sum(py::array_view values) { + std::int64_t out = 0; + for (auto value : values) { + out += value; + } + return out; + } + +This function has one parameter, ``values`` which is a view over the data being summed. +This parameter should be passed by value because it is only a view, and therefore small, like a :cpp:struct:`std::string_view`. + +From C++ +-------- + +:cpp:struct:`py::array_view` has an implicit constructor from any type that exposes both ``data()`` and ``size()`` member functions, like :cpp:struct:`std::vector`. +This means we can call ``simple_sum`` directly from C++, for example: + +.. code-block:: c++ + + std::vector vs(100); + std::iota(vs.begin(), vs.end(), 0); + + std::int64_t sum = simple_sum(vs); + +From Python +----------- + +To call ``simple_sum`` from Python, we must first use :cpp:func:`py::automethod` to adapt the function and then attach it to a module. +For example: + +.. code-block:: + + LIBPY_AUTOMODULE(libpy_tutorial, + arrays, + ({py::autofunction("simple_sum")})) + (py::borrowed_ref<>) { + return false; + } + +Now, we can import the function and pass it numpy arrays: + +.. ipython:: python + + import numpy as np + from libpy_tutorial.arrays import simple_sum + arr = np.arange(10); arr + simple_sum(arr) + +``py::array_view`` interface +============================ + +:cpp:struct:`py::ndarray_view` has the interface of a standard fixed-size C++ container, like :cpp:struct:`std::array`. +:cpp:struct:`py::ndarray_view` does have a few additions to the standard methods: + +Constructors +------------ + +- :cpp:func:`py::ndarray_view::from_buffer_protocol` +- :cpp:func:`py::ndarray_view::virtual_array` + +Extra Member Accessors +---------------------- + +- :cpp:func:`py::ndarray_view::shape` +- :cpp:func:`py::ndarray_view::strides` +- :cpp:func:`py::ndarray_view::buffer` +- :cpp:func:`py::ndarray_view::rank` +- :cpp:func:`py::ndarray_view::ssize` + +Contiguity +---------- + +Methods are helpers for checking if a view is over a contiguous array. + +- :cpp:func:`py::ndarray_view::is_c_contig` +- :cpp:func:`py::ndarray_view::is_f_contig` +- :cpp:func:`py::ndarray_view::is_contig` + +Derived Views +------------- + +- :cpp:func:`py::ndarray_view::freeze` +- :cpp:func:`py::ndarray_view::slice` diff --git a/docs/source/tutorial/functions.rst b/docs/source/tutorial/functions.rst new file mode 100644 index 00000000..47926578 --- /dev/null +++ b/docs/source/tutorial/functions.rst @@ -0,0 +1,105 @@ +========= +Functions +========= + +The simplest unit of code that can be exposed to Python from C++ is a function. +``libpy`` supports automatically converting C++ functions into Python functions by adapting the parameter types and return type. + +``libpy`` uses :cpp:func:`py::autofunction` to convert a C++ function into a Python function definition [#f1]_. +The result of :cpp:func:`py::autofunction` can be attached to a Python module object and made available to Python. + +A Simple C++ Function +===================== + +Let's start by writing a simple C++ function to expose to Python: + +.. code-block:: c++ + + double fma(double a, double b, double c) { + return a * b + c; + } + + +``fma`` is a standard C++ function with no knowledge of Python. + + +Adapting a Function +=================== + +To adapt ``fma`` into a Python function, we need to use :cpp:func:`py::autofunction`. + +.. code-block:: c++ + + PyMethodDef fma_methoddef = py::autofunction("fma"); + +:cpp:func:`py::autofunction` is a template function which takes as a template argument the C++ function to adapt. +:cpp:func:`py::autofunction` also takes a string which is the name of the function as it will be exposed to Python. +The Python function name does not need to match the C++ name. +:cpp:func:`py::autofunction` takes an optional second argument: a string to use as the Python docstring. +For example, a docstring could be added to ``fma`` with: + +.. code-block:: c++ + + PyMethodDef fma_methoddef = py::autofunction("fma", "Fused Multiply Add"); + +.. warning:: + + Currently the ``name`` and ``doc`` string parameters **must outlive** the resulting :c:struct:`PyMethodDef`. + In practice, this means it should be a static string, or string literal. + +Adding the Function to a Module +=============================== + +To use an adapted function from Python, it must be attached to a module so that it may be imported by Python code. +To create a Python method, we can use :c:macro:`LIBPY_AUTOMETHOD`. +:c:macro:`LIBPY_AUTOMETHOD` is a macro which takes in the package name, the module name, and the set of functions to add. +Following the call to :c:macro:`LIBPY_AUTOMETHOD`, we must provide a function which is called when the module is first imported. +To just add functions, our body can be a simple ``return false`` to indicate that no errors occurred. + +.. code-block:: c++ + + LIBPY_AUTOMODULE(libpy_tutorial, function, ({fma_methoddef})) + (py::borrowed_ref<>) { + return false; + } + +Building and Importing the Module +================================= + +To build a libpy extension, we can use ``setup.py`` and libpy's :class:`~libpy.build.LibpyExtension` class. + +In the ``setup.py``\'s ``setup`` call, we can add a list of ``ext_modules`` to be built: + +.. code-block:: python + + from libpy.build import LibpyExtension + + setup( + # ... + ext_modules=[ + LibpyExtension( + 'libpy_tutorial.function', + ['libpy_tutorial/function.cc'], + ), + ], + # ... + ) + +Now, the extension can be built with: + +.. code-block:: bash + + $ python setup.py build_ext --inplace + +Finally, the function can be imported and used from python: + + +.. ipython:: python + + import libpy # we need to ensure we import libpy before importing our extension + from libpy_tutorial.function import fma + fma(2.0, 3.0, 4.0) + +.. rubric:: Footnotes + +.. [#f1] :cpp:func:`py::autofunction` creates a :c:struct:`PyMethodDef` instance, which is not yet a Python object. diff --git a/docs/source/tutorial/libpy_tutorial/arrays.cc b/docs/source/tutorial/libpy_tutorial/arrays.cc index a69ea1bb..f5f262fc 100644 --- a/docs/source/tutorial/libpy_tutorial/arrays.cc +++ b/docs/source/tutorial/libpy_tutorial/arrays.cc @@ -7,7 +7,6 @@ #include namespace libpy_tutorial { - std::int64_t simple_sum(py::array_view values) { std::int64_t out = 0; for (auto value : values) { @@ -20,6 +19,13 @@ std::int64_t simple_sum_iterator(py::array_view values) { return std::accumulate(values.begin(), values.end(), 0); } +void negate_inplace(py::array_view values) { + std::transform(values.cbegin(), + values.cend(), + values.begin(), + [](std::int64_t v) { return -v; }); +} + bool check_prime(std::int64_t n) { if (n <= 3) { return n > 1; @@ -46,9 +52,9 @@ LIBPY_AUTOMODULE(libpy_tutorial, arrays, ({py::autofunction("simple_sum"), py::autofunction("simple_sum_iterator"), + py::autofunction("negate_inplace"), py::autofunction("is_prime")})) (py::borrowed_ref<>) { return false; } - } // namespace libpy_tutorial diff --git a/docs/source/tutorial/libpy_tutorial/function.cc b/docs/source/tutorial/libpy_tutorial/function.cc new file mode 100644 index 00000000..2ad2227c --- /dev/null +++ b/docs/source/tutorial/libpy_tutorial/function.cc @@ -0,0 +1,15 @@ +#include +#include + +namespace libpy_tutorial { +double fma(double a, double b, double c) { + return a * b + c; +} + +PyMethodDef fma_methoddef = py::autofunction("fma", "Fused Multiply Add"); + +LIBPY_AUTOMODULE(libpy_tutorial, function, ({fma_methoddef})) + (py::borrowed_ref<>) { + return false; +} +} // namespace libpy_tutorial diff --git a/docs/source/tutorial/setup.py b/docs/source/tutorial/setup.py index 29fc7e02..e6691b4c 100644 --- a/docs/source/tutorial/setup.py +++ b/docs/source/tutorial/setup.py @@ -70,5 +70,9 @@ def extension(*args, **kwargs): "libpy_tutorial.classes", ["libpy_tutorial/classes.cc"], ), + extension( + "libpy_tutorial.function", + ["libpy_tutorial/function.cc"], + ), ], ) diff --git a/include/libpy/automodule.h b/include/libpy/automodule.h index c9763da7..338fe90c 100644 --- a/include/libpy/automodule.h +++ b/include/libpy/automodule.h @@ -24,15 +24,8 @@ /** Define a Python module. - @param parent A symbol indicating the parent module. - @param name The leaf name of the module. - @param methods ({...}) list of objects representing the functions to add to the - module. Note this list must be surrounded by parentheses. - - ## Examples - - Create a module `my_package.submodule.my_module` with two functions `f` and - `g` and one type `T`. + For example, to create a module `my_package.submodule.my_module` with two functions + `f` and `g` and one type `T`: \code LIBPY_AUTOMODULE(my_package.submodule, @@ -43,7 +36,12 @@ py::borrowed_ref t = py::autoclass("T").new_().type(); return PyObject_SetAttrString(m.get(), "T", static_cast(t)); } - /endcode + \endcode + + @param parent A symbol indicating the parent module. + @param name The leaf name of the module. + @param methods `({...})` list of objects representing the functions to add to the + module. Note this list must be surrounded by parentheses. */ #define LIBPY_AUTOMODULE(parent, name, methods) \ bool _libpy_user_mod_init(py::borrowed_ref<>); \ diff --git a/libpy/build.py b/libpy/build.py index e1192904..427f9035 100644 --- a/libpy/build.py +++ b/libpy/build.py @@ -49,6 +49,8 @@ class LibpyExtension(setuptools.Extension, object): ---------- *args All positional arguments forwarded to :class:`setuptools.Extension`. + language_std : str, optional + The language standard to use. Defaults to ``c++17``. optlevel : int, optional The optimization level to forward to the C++ compiler. Defaults to 0. @@ -110,7 +112,6 @@ class LibpyExtension(setuptools.Extension, object): raise AssertionError('unknown compiler: %s' % _compiler) _base_flags = [ - '-std=gnu++17', '-pipe', '-fvisibility-inlines-hidden', '-DPY_MAJOR_VERSION=%d' % sys.version_info.major, @@ -123,7 +124,10 @@ class LibpyExtension(setuptools.Extension, object): def __init__(self, *args, **kwargs): kwargs['language'] = 'c++' + std = kwargs.pop('language_std', 'c++17') + libpy_extra_compile_args = self._base_flags.copy() + libpy_extra_compile_args.append('-std=%s' % std) libpy_extra_link_args = [] From ae9894f894519644f58d3a3822d617ffd342111b Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Thu, 9 Jul 2020 22:09:06 -0400 Subject: [PATCH 2/2] WIP --- docs/source/appendix.rst | 2 + docs/source/arrays.rst | 301 ++++++++++++++++++++++++++++++++ docs/source/index.rst | 3 +- docs/source/tutorial.rst | 1 - docs/source/tutorial/arrays.rst | 133 -------------- include/libpy/ndarray_view.h | 4 +- 6 files changed, 307 insertions(+), 137 deletions(-) create mode 100644 docs/source/arrays.rst delete mode 100644 docs/source/tutorial/arrays.rst diff --git a/docs/source/appendix.rst b/docs/source/appendix.rst index b9f3fbfd..5dbad479 100644 --- a/docs/source/appendix.rst +++ b/docs/source/appendix.rst @@ -298,6 +298,8 @@ Each of these types implements ``operator()`` to defer to the named operator whi .. doxygentypedef:: py::array_view +.. doxygenfunction:: py::for_each_unordered + Type Erased Views ----------------- diff --git a/docs/source/arrays.rst b/docs/source/arrays.rst new file mode 100644 index 00000000..f1656ebe --- /dev/null +++ b/docs/source/arrays.rst @@ -0,0 +1,301 @@ +====== +Arrays +====== + +To get the full power out of a C++ extension, you will often need to pass arrays of data between Python and C++. +Libpy has native support for integrating with numpy, the most popular ndarray library for Python. + +Libpy supports receiving arrays as views so that no data needs to be copied. +Libpy array views can also be const to guarantee that the underlying data isn't mutated. +Libpy also supports creating Numpy arrays as views over C++ containers without copying the underlying data. + +``py::array_view`` +================== + +Libpy can accept numpy arrays, or generally any buffer-like object, through a :cpp:class:`py::ndarray_view`. +:cpp:class:`py::ndarray_view` is a template type which takes as a parameter the C++ type of the elements of the array and the number of dimensions. +For example: ``py::ndarray_view`` is a view of a 3d array of signed 32 bit integers. +The type of the elements of a :cpp:class:`py::ndarray_view` are fixed at compile time, but the shape is determined at runtime. + +As a convenience, :cpp:type:`py::array_view` is an alias of :cpp:class:`py::ndarray_view` for one dimensional arrays. + +Shape and Strides +----------------- + +Like numpy, an array view is composed of three parts: + +- shape :: ``std::array`` +- strides :: ``std::array`` +- buffer :: ``(const) std::byte*`` + +The shape array contains the number of elements along each axis. +For example: ``{2, 3}`` would be an array with 2 rows and 3 columns. + +The strides array contains the number of bytes needed to move one step along each axis. +For example: given a ``{2, 3}`` shaped array of 4 byte elements, then strides of ``{12, 4}`` would be a C-contiguous array because the rows are contiguous. +Given the same ``{2, 3}`` shaped array of 4 byte elements, then strides of ``{4, 8}`` would be a Fortran-contiguous array because the rows are contiguous. + +The buffer must be a ``(const) std::byte*`` and not a ``(const) T*`` + +Non-contiguous views +-------------------- + +Array views do not need to view contiguous arrays. +For example, given a C-contiguous ``{4, 5}`` array of 2 byte values, we could take a view of first column by producing an array view with strides ``{10}``. + +Simple Array Input +================== + +Let's write function to sum an array: + +.. code-block:: c++ + + std::int64_t simple_sum(py::array_view values) { + std::int64_t out = 0; + for (auto value : values) { + out += value; + } + return out; + } + +This function has one parameter, ``values`` which is a view over the data being summed. +This parameter should be passed by value because it is only a view, and therefore small, like a :cpp:class:`std::string_view`. + +From C++ +-------- + +:cpp:type:`py::array_view` has an implicit constructor from any type that exposes both ``data()`` and ``size()`` member functions, like :cpp:class:`std::vector`. +This means we can call ``simple_sum`` directly from C++, for example: + +.. code-block:: c++ + + std::vector vs(100); + std::iota(vs.begin(), vs.end(), 0); + + std::int64_t sum = simple_sum(vs); + +From Python +----------- + +To call ``simple_sum`` from Python, we must first use :cpp:func:`py::automethod` to adapt the function and then attach it to a module. +For example: + +.. code-block:: + + LIBPY_AUTOMODULE(libpy_tutorial, + arrays, + ({py::autofunction("simple_sum")})) + (py::borrowed_ref<>) { + return false; + } + +Now, we can import the function and pass it numpy arrays: + +.. ipython:: python + + import numpy as np + from libpy_tutorial.arrays import simple_sum + arr = np.arange(10); arr + simple_sum(arr) + +Shallow Constness +================= + +:cpp:class:`py::ndarray_view` implements shallow constness. +Shallow constness means that a ``const py::ndarray_view`` allows mutation to the underlying data, but not mutation of what is being pointed to. +Shallow constness means that :cpp:class:`py::ndarray_view` acts like a pointer, not a reference. +One may have a ``const`` pointer to non ``const`` data. + +To create an immutable view, the ``const`` must be injected into the viewed type. +Instead of having a ``const`` view of ``int``, have a view of ``const int``. + +.. code-block:: c++ + + py::ndarray_view // mutable elements + const py::ndarray_view // mutable elements + py::ndarray_view // immutable elements + + +Freeze +------ + +Given a mutable view, the :cpp:func:`py::ndarray_view::freeze` member function returns an immutable view over the same data. +This is useful for ensuring that a particular component doesn't mutate a view that is otherwise mutable. +:cpp:func:`py::ndarray_view::freeze` exists for immutable views, but is a nop. + + +``py::array_view`` extended interface +===================================== + +:cpp:class:`py::ndarray_view` has the interface of a standard fixed-size C++ container, like :cpp:class:`std::array`. +:cpp:class:`py::ndarray_view` does have a few additions to the standard member functions: + +Constructors +------------ + +- :cpp:func:`py::ndarray_view::from_buffer_protocol` +- :cpp:func:`py::ndarray_view::virtual_array` + +Extra Member Accessors +---------------------- + +- :cpp:func:`py::ndarray_view::shape` +- :cpp:func:`py::ndarray_view::strides` +- :cpp:func:`py::ndarray_view::buffer` +- :cpp:func:`py::ndarray_view::rank` +- :cpp:func:`py::ndarray_view::ssize` + +Contiguity +---------- + +Member functions that are helpers for checking if a view is over a contiguous array. + +- :cpp:func:`py::ndarray_view::is_c_contig` +- :cpp:func:`py::ndarray_view::is_f_contig` +- :cpp:func:`py::ndarray_view::is_contig` + +Derived Views +------------- + +- :cpp:func:`py::ndarray_view::freeze` +- :cpp:func:`py::ndarray_view::slice` + +Free Functions +-------------- + +- :cpp:func:`py::for_each_unordered` + +Constructing Array Views +======================== + +Ndarray views may be constructed from C++ in a few ways. +The easiest way to get an ndarray view is to accept one as a parameter from a function which has been :cpp:func:`py::automethod` converted. +Libpy will take care of type and dimensionality checking and extracting the buffer from the underlying Python object. + +From C++ +-------- + +From Contiguous C++ Containers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +One dimensional array views, or :cpp:type:`py::array_view`, objects may be constructed from any C++ object that exposes both a ``data()`` and ``size()`` member functions. +``data()`` must return a ``T*`` which points to an array of ``T`` elements of size ``size()``. +Example containers that can be implicitly constructed from are :cpp:class:`std::vector` and :cpp:class:`std::array`. + +Example Usage +````````````` + +.. code-block:: c++ + + void from_vector() { + std::vector vec = {1, 2, 3}; + py::array_view view(vec); + } + + void from_array() { + std::array arr = {1, 2, 3}; + py::array_view view(arr); + } + +Low Level Constructor +~~~~~~~~~~~~~~~~~~~~~ + +If one wishes to construct a view from C++ directly, the most fundamental constructor takes the buffer as a ``(const) std::byte*``, the shape array, and the strides array. +It is the user's responsibility to ensure that the buffer is compatible with the provided shape and strides, no checking will or can be done. + +From Buffer-like Objects +~~~~~~~~~~~~~~~~~~~~~~~~ + +To construct an array view from a Python object that exports the buffer protocol, like a :class:`memoryview` or numpy array, there is a static member function :cpp:func:`py::ndarray_view::from_buffer_protocol`. +Unlike a normal constructor, :cpp:func:`py::ndarray_view::from_buffer_protocol` returns a tuple of two parts: the array view instance and a :cpp:type:`py::buffer`. +The :cpp:type:`py::buffer` is an RAII object which manages the lifetime of the underlying buffer which the view is over. +The returned view is only valid as long as the paired :cpp:type:`py::buffer` is alive. +Accessing through the view outside the lifetime of the :cpp:type:`py::buffer`c may trigger a use after free and is undefined behavior. + +:cpp:func:`py::ndarray_view::from_buffer_protocol` will check that the runtime type of the Python buffer matches the static type of the C++ array view. +:cpp:func:`py::ndarray_view::from_buffer_protocol` will also check that the runtime dimensionality of the Python buffer matches the static dimensionality of the C++ array view. + +Virtual Array Views +~~~~~~~~~~~~~~~~~~~ + +A virtual array view is a scalar which is broadcasted to present as an array view. +Concretely, a virtual array uses the ``buffer`` member to hold a pointer to a single value, and has strides of all zeros. +By setting all of the strides to zero, this means that the single scalar can satisfy any shape. + +A virtual array view is useful when one must satisfy and interface that requires an array view but would like to pass a constant value. +A virtual array view is considerably more efficient than allocating an array and filling it with a constant. +No memory must be allocated, and each access will go to the same cache line. + +Because all elements of the view share the same underlying memory, mutable virtual arrays can have unexpected results. +If any value in the array view is mutated, all of the elements would change. +This can have unexpected consequences when passing the views to functions that are not prepared for that behavior. +For this reason, it is recommended to only use const virtual array views. + +Virtual array views do not copy nor move from the element being viewed. +For that reason, the view must not outlive the element being broadcasted. + +Example Usage +````````````` + +.. code-block:: c++ + + // Library code + + /** A function which adds two array views, storing the result in the first + array view. + */ + void add_inplace(py::array_view a, py::array_view b) { + std::transform(a.cbegin(), a.cend(), b.cbegin(), a.begin(), std::plus<>{}); + } + + // User code + + /** The user defined function which wants to call `add_inplace` with a + scalar. + */ + void f(py::array_view a) { + int rhs = 5; + auto rhs_view = py::array_view::virtual_array(rhs, a.shape()); + + // `rhs_view` points to the same data as `rhs` + assert(rhs_view.buffer() == reinterpret_cast(&rhs)); + + add_inplace(a, rhs_view); + + // ... + } + +Here, it is critical not to use ``rhs_view`` after ``rhs`` has gone out of scope because the buffer points to the memory owned by ``rhs``. + +Type Erased Views +================= + +:cpp:class:`py::ndarray_view` normally have a static type for the elements; however, Python users of numpy arrays might not always think of arrays in this way. +Libpy currently only supports exporting a single overload of a function, so some functions which could be written generically need to have a single signature which can accept arrays of any type. +In addition to the restriction of having a single overload exposed, for some functions, adding a lot of template expansions to have static types doesn't meaningfully improve the performance to justify the increased compile times. + +To provide static type-erased values, there are types :cpp:class:`py::any_ref` and :cpp:class:`py::any_cref`. +:cpp:class:`py::any_ref` values act like references, and :cpp:class:`py::any_cref` act like ``const`` references. +Unlike a ``void*``, :cpp:class:`py::any_ref` and :cpp:class:`py::any_cref` hold a virtual method table which implements some basic functionality. +The vtable for both type-erased reference types is a :cpp:class:`py::any_vtable`. +:cpp:class:`py::any_vtable` supports constructing new values, copying, moving, checking equality, and getting the numpy dtype for the type. +:cpp:class:`py::any_vtable` can also provide information about the type like the size and alignment. + +``py::array_view`` and ``py::array_view`` have more specific meaning than "view of an array of any ref objects". +Instead, ``py::array_view`` and ``py::array_view`` are always homogeneous, meaning all of the elements are the same type. +``py::array_view`` and ``py::array_view`` have the following members: + +- shape :: ``std::array`` +- strides :: ``std::array`` +- buffer :: ``(const) std::byte*`` +- vtable :: :cpp:class:`py::any_vtable` + +The shape and strides are the same as a normal :cpp:class:`py::ndarray`. +The buffer is now a pointer to an untyped block of data which should be interpreted based on the vtable. +The vtable member encodes the type of the elements in the array and provides access to the operations on the elements. + +Type Casting +------------ + +For performance reasons, it is still useful to convert to a statically typed array view sometimes. +There is a :cpp:func:`py::ndarray_view::cast` template member function which diff --git a/docs/source/index.rst b/docs/source/index.rst index 71f55cfc..25eb95c5 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,8 +11,9 @@ Welcome to libpy's documentation! :caption: Contents: shock-and-awe - tutorial install + tutorial + arrays appendix diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 8ebbb51b..dba64d23 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -6,4 +6,3 @@ Tutorial :caption: Contents: tutorial/functions - tutorial/arrays diff --git a/docs/source/tutorial/arrays.rst b/docs/source/tutorial/arrays.rst deleted file mode 100644 index d565fa95..00000000 --- a/docs/source/tutorial/arrays.rst +++ /dev/null @@ -1,133 +0,0 @@ -====== -Arrays -====== - -To get the full power out of a C++ extension, you will often need to pass arrays of data between Python and C++. -Libpy has native support for integrating with numpy, the most popular ndarray library for Python. - -Libpy supports receiving arrays as views so that no data needs to be copied. -Libpy array views can also be const to guarantee that the underlying data isn't mutated. -Libpy also supports creating Numpy arrays as views over C++ containers without copying the underlying data. - -``py::array_view`` -================== - -Libpy can accept numpy arrays, or generally any buffer-like object, through a :cpp:struct:`py::ndarray_view`. -:cpp:struct:`py::ndarray_view` is a template type which takes as a parameter the C++ type of the elements of the array and the number of dimensions. -For example: ``py::ndarray_view`` is a view of a 3d array of signed 32 bit integers. -The type of the elements of a :cpp:struct:`py::ndarray_view` are fixed at compile time, but the shape is determined at runtime. - -As a convenience, :cpp:struct:`py::array_view` is an alias of :cpp:struct:`py::ndarray_view` for 1 dimensional arrays. - -Shape and Strides ------------------ - -Like numpy, an array view is composed of three parts: - -- shape :: ``std::array`` -- strides :: ``std::array`` -- buffer :: ``(const) std::byte*`` - -The shape array contains the number of elements along each axis. -For example: ``{2, 3}`` would be an array with 2 rows and 3 columns. - -The strides array contains the number of bytes needed to move one step along each axis. -For example: given a ``{2, 3}`` shaped array of 4 byte elements, then strides of ``{12, 4}`` would be a C-contiguous array because the rows are contiguous. -Given the same ``{2, 3}`` shaped array of 4 byte elements, then strides of ``{4, 8}`` would be a Fortran-contiguous array because the rows are contiguous. - -Non-contiguous views --------------------- - -Array views do not need to view contiguous arrays. -For example, given a C-contiguous ``{4, 5}`` array of 2 byte values, we could take a view of first column by producing an array view with strides ``{10}``. - -Simple Array Input -================== - -Let's write function to sum an array: - -.. code-block:: c++ - - std::int64_t simple_sum(py::array_view values) { - std::int64_t out = 0; - for (auto value : values) { - out += value; - } - return out; - } - -This function has one parameter, ``values`` which is a view over the data being summed. -This parameter should be passed by value because it is only a view, and therefore small, like a :cpp:struct:`std::string_view`. - -From C++ --------- - -:cpp:struct:`py::array_view` has an implicit constructor from any type that exposes both ``data()`` and ``size()`` member functions, like :cpp:struct:`std::vector`. -This means we can call ``simple_sum`` directly from C++, for example: - -.. code-block:: c++ - - std::vector vs(100); - std::iota(vs.begin(), vs.end(), 0); - - std::int64_t sum = simple_sum(vs); - -From Python ------------ - -To call ``simple_sum`` from Python, we must first use :cpp:func:`py::automethod` to adapt the function and then attach it to a module. -For example: - -.. code-block:: - - LIBPY_AUTOMODULE(libpy_tutorial, - arrays, - ({py::autofunction("simple_sum")})) - (py::borrowed_ref<>) { - return false; - } - -Now, we can import the function and pass it numpy arrays: - -.. ipython:: python - - import numpy as np - from libpy_tutorial.arrays import simple_sum - arr = np.arange(10); arr - simple_sum(arr) - -``py::array_view`` interface -============================ - -:cpp:struct:`py::ndarray_view` has the interface of a standard fixed-size C++ container, like :cpp:struct:`std::array`. -:cpp:struct:`py::ndarray_view` does have a few additions to the standard methods: - -Constructors ------------- - -- :cpp:func:`py::ndarray_view::from_buffer_protocol` -- :cpp:func:`py::ndarray_view::virtual_array` - -Extra Member Accessors ----------------------- - -- :cpp:func:`py::ndarray_view::shape` -- :cpp:func:`py::ndarray_view::strides` -- :cpp:func:`py::ndarray_view::buffer` -- :cpp:func:`py::ndarray_view::rank` -- :cpp:func:`py::ndarray_view::ssize` - -Contiguity ----------- - -Methods are helpers for checking if a view is over a contiguous array. - -- :cpp:func:`py::ndarray_view::is_c_contig` -- :cpp:func:`py::ndarray_view::is_f_contig` -- :cpp:func:`py::ndarray_view::is_contig` - -Derived Views -------------- - -- :cpp:func:`py::ndarray_view::freeze` -- :cpp:func:`py::ndarray_view::slice` diff --git a/include/libpy/ndarray_view.h b/include/libpy/ndarray_view.h index a03eeef0..a4f473f7 100644 --- a/include/libpy/ndarray_view.h +++ b/include/libpy/ndarray_view.h @@ -265,7 +265,7 @@ class ndarray_view { return is_c_contig() || is_f_contig(); } - /** The underlying buffer of characters for this string array. + /** Get the underlying buffer for this view. */ buffer_type buffer() const { return m_buffer; @@ -875,7 +875,7 @@ class any_ref_ndarray_view { return {m_buffer, m_shape, m_strides}; } - /** The underlying buffer of characters for this string array. + /** Get the underlying buffer for this view. */ auto buffer() const { return m_buffer;