Simple prehash interning + some convenience functionality

demo/customslots.pxd

@@ -5,9 +5,9 @@ cdef extern from "customslots.h":
         void *pointer
         Py_ssize_t objoffset
         uintptr_t flags
-    
+
     ctypedef struct PyCustomSlot:
-        uintrptr_t id
+        uintptr_t id
         pyx_data data
 
     int PyCustomSlots_Check(obj)

extensibletype/extensibletype.pxd

@@ -0,0 +1,27 @@
+cdef extern from "pstdint.h":
+    ctypedef unsigned int uint32_t
+    ctypedef unsigned long long uint64_t
+    ctypedef unsigned short uint16_t
+    ctypedef unsigned char uint8_t
+    ctypedef uint64_t uintptr_t
+
+cdef extern from "perfecthash.h":
+    ctypedef struct PyCustomSlots_Entry:
+        uint64_t id
+        void *ptr
+
+    ctypedef struct PyCustomSlots_Table:
+        uint64_t flags
+        uint64_t m_f, m_g
+        PyCustomSlots_Entry *entries
+        uint16_t n, b
+        uint8_t r
+
+    ctypedef struct PyCustomSlots_Table_64_64:
+        PyCustomSlots_Table base
+        uint16_t d[64]
+        PyCustomSlots_Entry entries_mem[64]
+
+
+    int PyCustomSlots_PerfectHash(PyCustomSlots_Table *table, uint64_t *hashes)
+

extensibletype/extensibletype.pyx

@@ -1,42 +1,30 @@
 cimport numpy as cnp
 import numpy as np
 
-cdef extern from "stdint.h":
-    ctypedef unsigned int uint32_t
-    ctypedef unsigned long long uint64_t
-    ctypedef unsigned short uint16_t
-    ctypedef unsigned char uint8_t
-    ctypedef uint64_t uintptr_t
+import hashlib
 
 cdef extern from "perfecthash.h":
-    ctypedef struct PyCustomSlots_Entry:
-        char *id
-        uintptr_t flags
-        void *ptr
-
-    ctypedef struct PyCustomSlots_Table:
-        uint64_t flags
-        uint64_t m_f, m_g
-        PyCustomSlots_Entry *entries
-        uint16_t n, b
-        uint8_t r
-
-    ctypedef struct PyCustomSlots_Table_64_64:
-        PyCustomSlots_Table base
-        uint16_t d[64]
-        PyCustomSlots_Entry entries_mem[64]
-
-
-    int PyCustomSlots_PerfectHash(PyCustomSlots_Table *table, uint64_t *hashes)
     void _PyCustomSlots_bucket_argsort(uint16_t *p, uint8_t *binsizes,
-                                       uint8_t *number_of_bins_by_size)
+                                       uint16_t *number_of_bins_by_size)
 
 def bucket_argsort(cnp.ndarray[uint16_t, mode='c'] p,
                    cnp.ndarray[uint8_t, mode='c'] binsizes,
-                   cnp.ndarray[uint8_t, mode='c'] number_of_bins_by_size):
+                   cnp.ndarray[uint16_t, mode='c'] number_of_bins_by_size):
     _PyCustomSlots_bucket_argsort(&p[0], &binsizes[0],
                                   &number_of_bins_by_size[0])
 
+def get_random_hashes(rng, nitems):
+    return rng.randint(-2**31, 2**31-1, size=nitems).astype(np.uint64)
+
+def draw_hashes(rng, nitems):
+    assert sizeof(long) >= 4
+
+    hashes = get_random_hashes(rng, nitems)
+    hashes <<= 32
+    hashes |= get_random_hashes(rng, nitems)
+
+    return hashes
+
 def perfect_hash(cnp.ndarray[uint64_t] hashes, int repeat=1):
     """Used for testing. Takes the hashes as input, and returns
        a permutation array and hash parameters:
@@ -49,8 +37,7 @@ def perfect_hash(cnp.ndarray[uint64_t] hashes, int repeat=1):
     table.base.b = 64
     table.base.entries = &table.entries_mem[0]
     for i in range(64):
-        table.entries_mem[i].id = NULL
-        table.entries_mem[i].flags = i
+        table.entries_mem[i].id = hashes[i]
         table.entries_mem[i].ptr = NULL
 
     cdef int r
@@ -61,38 +48,38 @@ def perfect_hash(cnp.ndarray[uint64_t] hashes, int repeat=1):
     p = np.zeros(64, dtype=np.uint16)
 
     for i in range(64):
-        p[i] = table.entries_mem[i].flags
+        p[i] = table.entries_mem[i].id & 0xFF
         d[i] = table.d[i]
 
     return p, table.base.r, table.base.m_f, table.base.m_g, d
 
-cdef extern from "md5sum.h":
-    ctypedef struct MD5_CTX:
-        uint32_t i[2]
-        uint32_t buf[4]
-        unsigned char in_ "in"[64]
-        unsigned char digest[16]
-
-    void MD5Init(MD5_CTX *mdContext)
-    void MD5Update(MD5_CTX *mdContext, unsigned char *inBuf,
-                   unsigned int inLen)
-    void MD5Final(MD5_CTX *mdContext)
-    
-cdef extern from "hash.h":
-    uint64_t hash_crapwow64(unsigned char *buf, uint64_t len, uint64_t seed)
-
-def crapwowbench(int repeat=1):
-    cdef int r
-    cdef MD5_CTX ctx
-    for r in range(repeat):
-        hash_crapwow64("asdf", 4, 0xf123456781234567)
-
-
-def md5bench(int repeat=1):
-    cdef int r
-    cdef MD5_CTX ctx
-    for r in range(repeat):
-        MD5Init(&ctx)
-        MD5Update(&ctx, "asdf", 4)
-        MD5Final(&ctx)
+#cdef extern from "md5sum.h":
+#    ctypedef struct MD5_CTX:
+#        uint32_t i[2]
+#        uint32_t buf[4]
+#        unsigned char in_ "in"[64]
+#        unsigned char digest[16]
+#
+#    void MD5Init(MD5_CTX *mdContext)
+#    void MD5Update(MD5_CTX *mdContext, unsigned char *inBuf,
+#                   unsigned int inLen)
+#    void MD5Final(MD5_CTX *mdContext)
+#
+#cdef extern from "hash.h":
+#    uint64_t hash_crapwow64(unsigned char *buf, uint64_t len, uint64_t seed)
+#
+#def crapwowbench(int repeat=1):
+#    cdef int r
+#    cdef MD5_CTX ctx
+#    for r in range(repeat):
+#        hash_crapwow64("asdf", 4, 0xf123456781234567)
+#
+#
+#def md5bench(int repeat=1):
+#    cdef int r
+#    cdef MD5_CTX ctx
+#    for r in range(repeat):
+#        MD5Init(&ctx)
+#        MD5Update(&ctx, "asdf", 4)
+#        MD5Final(&ctx)
 

extensibletype/intern.pxd

@@ -0,0 +1,17 @@
+from extensibletype cimport *
+
+cdef extern from "Python.h":
+    ctypedef unsigned int Py_uintptr_t
+
+cdef extern from *:
+    ctypedef char *string_t "const char *"
+
+cdef extern from "globalinterning.h":
+    ctypedef void *intern_table_t
+
+    intern_table_t *intern_create_table(intern_table_t *table) except NULL
+    void intern_destroy_table(intern_table_t *table)
+    uint64_t intern_key(intern_table_t *table, string_t key) except? 0
+
+    int PyIntern_Initialize() except -1
+    uint64_t PyIntern_AddKey(string_t key) except? 0

extensibletype/intern.pyx

@@ -0,0 +1,20 @@
+def global_intern(bytes key):
+    return PyIntern_AddKey(key)
+
+def global_intern_initialize():
+    PyIntern_Initialize()
+
+cdef class InternTable(object):
+    "Wrap intern tables (intern_table_t)"
+
+    cdef intern_table_t _table
+    cdef intern_table_t *table
+
+    def __init__(self):
+        self.table = intern_create_table(&self._table)
+
+    def __dealloc__(self):
+        intern_destroy_table(self.table)
+
+    def intern(self, bytes key):
+        return intern_key(self.table, key)

extensibletype/methodtable.pyx

@@ -0,0 +1,183 @@
+from libc cimport stdlib
+cimport numpy as cnp
+import numpy as np
+
+from extensibletype cimport *
+from . import extensibletype
+
+import intern
+
+def roundup(x):
+    "Round up to a power of two"
+    x -= 1
+    x |= x >> 1
+    x |= x >> 2
+    x |= x >> 4
+    x |= x >> 8
+    x |= x >> 16
+    x |= x >> 32
+    x += 1
+    return x
+
+class HashingError(Exception):
+    """
+    Raised when we can't create a perfect hash-based function table.
+    """
+
+cdef PyCustomSlots_Table *allocate_hash_table(uint16_t size) except NULL:
+    cdef PyCustomSlots_Table *table
+    cdef uint16_t nbins
+
+    size = roundup(size)
+    assert size * 4 <= 0xFFFF, hex(size)
+    nbins = size * 4
+
+    table = <PyCustomSlots_Table *> stdlib.calloc(
+        1, sizeof(PyCustomSlots_Table) + sizeof(uint16_t) * nbins +
+           sizeof(PyCustomSlots_Entry) * size)
+
+    if table == NULL:
+        raise MemoryError
+
+    table.n = size
+    table.b = nbins
+    table.flags = 0
+
+    assert table.b >= table.n, (table.b, table.n, nbins)
+
+    table.entries = <PyCustomSlots_Entry *> (
+        (<char *> table) + sizeof(PyCustomSlots_Table) +
+        nbins * sizeof(uint16_t))
+
+    return table
+
+def make_bytes(s):
+    if isinstance(s, str):
+        # Python 3
+        s = s.encode("ascii")
+
+    return s
+
+cdef class Hasher(object):
+    """
+    Generate a globally unique hashes for signature strings.
+    """
+
+    def hash_signature(self, signature):
+        cdef uint64_t hashvalue
+        # cdef bytes md5 = hashlib.md5(signature).digest()
+        # (&hashvalue)[0] = (<uint64_t *> <char *> md5)[0]
+
+        hashvalue = intern.global_intern(make_bytes(signature))
+        return hashvalue
+
+
+cdef class PerfectHashMethodTable(object):
+    """
+    Simple wrapper for hash-based virtual method tables.
+    """
+
+    cdef PyCustomSlots_Table *table
+    cdef uint16_t *displacements
+    cdef Hasher hasher
+
+    cdef object id_to_signature, signatures
+
+    def __init__(self, hasher):
+        self.hasher = hasher
+        # For debugging
+        self.id_to_signature = {}
+
+    def generate_table(self, n, ids, flags, funcs, method_names=None):
+        cdef Py_ssize_t i
+        cdef cnp.ndarray[uint64_t] hashes
+
+        self.table = allocate_hash_table(n)
+        self.displacements = <uint16_t *> (<char *> self.table +
+                                               sizeof(PyCustomSlots_Table))
+
+        hashes = np.zeros(self.table.n, dtype=np.uint64)
+
+        intern.global_intern_initialize()
+
+        # Initialize hash table entries, build hash ids
+        assert len(ids) == len(flags) == len(funcs)
+
+        for i, (signature, flag, func) in enumerate(zip(ids, flags, funcs)):
+            id = self.hasher.hash_signature(signature)
+
+            self.table.entries[i].id = id
+            self.table.entries[i].ptr = <void *> <uintptr_t> func
+
+            hashes[i] = id
+            self.id_to_signature[id] = signature
+
+
+        hashes[n:self.table.n] = extensibletype.draw_hashes(np.random,
+                                                            self.table.n - n)
+        # print "n", n, "table.n", self.table.n, "table.b", self.table.b
+        assert len(np.unique(hashes)) == len(hashes)
+
+        # print "-----------------------"
+        # print self
+        # print "-----------------------"
+
+        assert self.table.b >= self.table.n, (self.table.b, self.table.n)
+
+        # Perfect hash our table
+        if PyCustomSlots_PerfectHash(self.table, &hashes[0]) < 0:
+            # TODO: sensible error messages
+            raise HashingError(
+                "Unable to create perfect hash table for table: %s" % self)
+
+        for i, signature in enumerate(ids):
+            assert self.find_method(signature) is not None, (i, signature)
+
+        # For debugging
+        self.signatures = ids
+
+    def find_method(self, signature):
+        """
+        Find method of the given signature. Use from non-performance
+        critical code.
+        """
+        cdef uint64_t prehash = intern.global_intern(make_bytes(signature))
+
+        assert 0 <= self.displacements[prehash & self.table.m_g] < self.table.b
+        cdef uint64_t idx = (((prehash >> self.table.r) & self.table.m_f) ^
+                             self.displacements[prehash & self.table.m_g])
+
+        assert 0 <= idx < self.size, (idx, self.size)
+
+        if self.table.entries[idx].id != prehash:
+            return None
+
+        return (<uintptr_t> self.table.entries[idx].ptr,
+                self.table.entries[idx].id & 0xFF)
+
+    def __str__(self):
+        buf = ["PerfectHashMethodTable("]
+        for i in range(self.table.n):
+            id = self.table.entries[i].id
+            ptr = <uintptr_t> self.table.entries[i].ptr
+            sig = self.id_to_signature.get(id, "<empty>")
+            s = "    id: 0x%-16x  funcptr: %20d  signature: %s" % (id, ptr, sig)
+            buf.append(s)
+
+        buf.append(")")
+
+        return "\n".join(buf)
+
+    def __dealloc__(self):
+        # stdlib.free(self.table)
+        # self.table = NULL
+        pass
+
+    property table_ptr:
+        def __get__(self):
+            return <uintptr_t> self.table
+
+    property size:
+        def __get__(self):
+            return self.table.n
+