WIP: NRT

.circleci/config.yml

@@ -17,7 +17,6 @@ jobs:
       #     key: conda-pkgs-{{ checksum "environment.yml" }}
       #     paths:
       #       - /opt/conda/pkgs
-      - run: source activate numba-xnd && cd numba && python setup.py develop
       - run: source activate numba-xnd && python setup.py develop
       - run: source activate numba-xnd && coverage run --include "numba_xnd/*" -m unittest -v
       - run: source activate numba-xnd && codecov

.numba_config.yaml

@@ -0,0 +1,3 @@
+developer_mode: 1
+color_scheme: dark_bg
+dump_optimized: 1

README.md

@@ -43,9 +43,6 @@ assert simple_matrix_multiply(a, b) == c
 ```bash
 conda env create
 conda activate numba-xnd
-cd numba
-python setup.py develop
-cd ..
 python setup.py develop
 # ready to run project scripts
 ```

environment.yml

@@ -11,3 +11,9 @@ dependencies:
   - xnd
   - coverage
   - codecov
+  - pyyaml # for numba config parsing
+  - ipython # for developing
+  - numba
+  - pytest
+  - pytest-xdist
+  - colorama

notebooks/12. Numba Tuples.ipynb

@@ -0,0 +1,189 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's try to get tuples to work with range https://github.com/numba/numba/issues/2771\n",
+    "\n",
+    "We can do this with recursion"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "I wanna be able to construct a tuple from this signature:\n",
+    "\n",
+    "`create_tuple(lambda i: ..., n)`\n",
+    "\n",
+    "So that:\n",
+    "\n",
+    "```python\n",
+    "create_tuple(lambda i: i**2, 3) == (0, 1, 4)\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from numba import njit"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "TypingError",
+     "evalue": "Failed at nopython (nopython frontend)\nFailed at nopython (nopython frontend)\nFailed at nopython (nopython frontend)\nCan't unify return type from the following types: tuple(int64 x 1), tuple(int64 x 2)\nReturn of: IR name '$12.5', type 'tuple(int64 x 1)', location: \nFile \"<ipython-input-3-203536fd220c>\", line 4:\ndef create_tuple(f, n, i=0):\n    <source elided>\n    if i == n - 1:\n        return (f(i),)\n        ^\nReturn of: IR name '$22.13', type 'tuple(int64 x 2)', location: \nFile \"<ipython-input-3-203536fd220c>\", line 5:\ndef create_tuple(f, n, i=0):\n    <source elided>\n        return (f(i),)\n    return (f(i),) + create_tuple(f, n, i + 1)\n    ^\n[1] During: resolving callee type: recursive(type(CPUDispatcher(<function create_tuple at 0x113e94598>)))\n[2] During: typing of call at <ipython-input-3-203536fd220c> (5)\n\n\nFile \"<ipython-input-3-203536fd220c>\", line 5:\ndef create_tuple(f, n, i=0):\n    <source elided>\n        return (f(i),)\n    return (f(i),) + create_tuple(f, n, i + 1)\n    ^\n\n[1] During: resolving callee type: recursive(type(CPUDispatcher(<function create_tuple at 0x113e94598>)))\n[2] During: typing of call at <ipython-input-3-203536fd220c> (5)\n\n\nFile \"<ipython-input-3-203536fd220c>\", line 5:\ndef create_tuple(f, n, i=0):\n    <source elided>\n        return (f(i),)\n    return (f(i),) + create_tuple(f, n, i + 1)\n    ^\n\n[1] During: resolving callee type: type(CPUDispatcher(<function create_tuple at 0x113e94598>))\n[2] During: typing of call at <ipython-input-3-203536fd220c> (14)\n\n\nFile \"<ipython-input-3-203536fd220c>\", line 14:\ndef test():\n    return create_tuple(f, 3)\n    ^\n\nThis is not usually a problem with Numba itself but instead often caused by\nthe use of unsupported features or an issue in resolving types.\n\nTo see Python/NumPy features supported by the latest release of Numba visit:\nhttp://numba.pydata.org/numba-doc/dev/reference/pysupported.html\nand\nhttp://numba.pydata.org/numba-doc/dev/reference/numpysupported.html\n\nFor more information about typing errors and how to debug them visit:\nhttp://numba.pydata.org/numba-doc/latest/user/troubleshoot.html#my-code-doesn-t-compile\n\nIf you think your code should work with Numba, please report the error message\nand traceback, along with a minimal reproducer at:\nhttps://github.com/numba/numba/issues/new\n",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypingError\u001b[0m                               Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-3-203536fd220c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mcreate_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mtest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/p/numba-xnd/numba/numba/dispatcher.py\u001b[0m in \u001b[0;36m_compile_for_args\u001b[0;34m(self, *args, **kws)\u001b[0m\n\u001b[1;32m    347\u001b[0m                 \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpatch_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 349\u001b[0;31m             \u001b[0merror_rewrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'typing'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    350\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mUnsupportedError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m             \u001b[0;31m# Something unsupported is present in the user code, add help info\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/p/numba-xnd/numba/numba/dispatcher.py\u001b[0m in \u001b[0;36merror_rewrite\u001b[0;34m(e, issue_type)\u001b[0m\n\u001b[1;32m    314\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    315\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 316\u001b[0;31m                 \u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m         \u001b[0margtypes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/p/numba-xnd/numba/numba/six.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m    656\u001b[0m             \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    657\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 658\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    659\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    660\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mTypingError\u001b[0m: Failed at nopython (nopython frontend)\nFailed at nopython (nopython frontend)\nFailed at nopython (nopython frontend)\nCan't unify return type from the following types: tuple(int64 x 1), tuple(int64 x 2)\nReturn of: IR name '$12.5', type 'tuple(int64 x 1)', location: \nFile \"<ipython-input-3-203536fd220c>\", line 4:\ndef create_tuple(f, n, i=0):\n    <source elided>\n    if i == n - 1:\n        return (f(i),)\n        ^\nReturn of: IR name '$22.13', type 'tuple(int64 x 2)', location: \nFile \"<ipython-input-3-203536fd220c>\", line 5:\ndef create_tuple(f, n, i=0):\n    <source elided>\n        return (f(i),)\n    return (f(i),) + create_tuple(f, n, i + 1)\n    ^\n[1] During: resolving callee type: recursive(type(CPUDispatcher(<function create_tuple at 0x113e94598>)))\n[2] During: typing of call at <ipython-input-3-203536fd220c> (5)\n\n\nFile \"<ipython-input-3-203536fd220c>\", line 5:\ndef create_tuple(f, n, i=0):\n    <source elided>\n        return (f(i),)\n    return (f(i),) + create_tuple(f, n, i + 1)\n    ^\n\n[1] During: resolving callee type: recursive(type(CPUDispatcher(<function create_tuple at 0x113e94598>)))\n[2] During: typing of call at <ipython-input-3-203536fd220c> (5)\n\n\nFile \"<ipython-input-3-203536fd220c>\", line 5:\ndef create_tuple(f, n, i=0):\n    <source elided>\n        return (f(i),)\n    return (f(i),) + create_tuple(f, n, i + 1)\n    ^\n\n[1] During: resolving callee type: type(CPUDispatcher(<function create_tuple at 0x113e94598>))\n[2] During: typing of call at <ipython-input-3-203536fd220c> (14)\n\n\nFile \"<ipython-input-3-203536fd220c>\", line 14:\ndef test():\n    return create_tuple(f, 3)\n    ^\n\nThis is not usually a problem with Numba itself but instead often caused by\nthe use of unsupported features or an issue in resolving types.\n\nTo see Python/NumPy features supported by the latest release of Numba visit:\nhttp://numba.pydata.org/numba-doc/dev/reference/pysupported.html\nand\nhttp://numba.pydata.org/numba-doc/dev/reference/numpysupported.html\n\nFor more information about typing errors and how to debug them visit:\nhttp://numba.pydata.org/numba-doc/latest/user/troubleshoot.html#my-code-doesn-t-compile\n\nIf you think your code should work with Numba, please report the error message\nand traceback, along with a minimal reproducer at:\nhttps://github.com/numba/numba/issues/new\n"
+     ]
+    }
+   ],
+   "source": [
+    "@njit\n",
+    "def create_tuple(f, n, i=0):\n",
+    "    if i == n - 1:\n",
+    "        return (f(i),)\n",
+    "    return (f(i),) + create_tuple(f, n, i + 1)\n",
+    "\n",
+    "\n",
+    "@njit\n",
+    "def f(i):\n",
+    "    return i**2\n",
+    "\n",
+    "@njit\n",
+    "def test():\n",
+    "    return create_tuple(f, 3)\n",
+    "\n",
+    "test()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "OK that doesn't work... Instead we create $n$ dispatchers, one for each tuple size."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "((0, 1, 8, 27), (0, 1, 4, 9))"
+      ]
+     },
+     "execution_count": 54,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def create_tuple_creator(f, n):\n",
+    "    assert n > 0\n",
+    "    f = njit(f)\n",
+    "    @njit\n",
+    "    def creator(args):\n",
+    "        return (f(0, *args),)\n",
+    "    for i in range(1, n):\n",
+    "        # need to pass in creator and i to lambda to capture in scope\n",
+    "        @njit\n",
+    "        def creator(args, creator=creator, i=i):\n",
+    "            return creator(args) + (f(i, *args),)\n",
+    "    return njit(lambda *args: creator(args))\n",
+    "\n",
+    "creator = create_tuple_creator(lambda i, j: i**j, 4)\n",
+    "\n",
+    "@njit\n",
+    "def test():\n",
+    "    return creator(3), creator(2)\n",
+    "\n",
+    "test()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now let's re-create some of the examples in the numba issue: https://github.com/numba/numba/issues/2771#issuecomment-368620310"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)\n",
+      "(0, 2, 4, 6, 8, 10, 12, 14, 16, 18)\n",
+      "((True, False), (True, False), (True, False), (True, False), (True, False), (True, False), (True, False), (True, False), (True, False), (True, False))\n"
+     ]
+    }
+   ],
+   "source": [
+    "range_10 = create_tuple_creator(lambda i: i, 10)\n",
+    "range_10_x = create_tuple_creator(lambda i, x: i*x, 10)\n",
+    "true_10 = create_tuple_creator(lambda _: True, 10)\n",
+    "false_10 = create_tuple_creator(lambda _: False, 10)\n",
+    "zip_10 = create_tuple_creator(lambda i, l, r: (l[i], r[i]), 10)\n",
+    "\n",
+    "@njit\n",
+    "def foo(x):\n",
+    "    print(range_10()) # tuple([i for i in range(10)])\n",
+    "    print(range_10_x(x)) # tuple([i*x for i in range(10)])\n",
+    "    print(zip_10(true_10(), false_10())),  # tuple(zip((True,)*10, (False,)*10))\n",
+    "\n",
+    "    \n",
+    "foo(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}