Merge pull request #6 from static-frame/5/unicode-eq

Fix `memcmp` usage to provide units in bytes

Author: flexatone

arraymap.c

@@ -42,6 +42,7 @@ typedef struct TableElement{
 # define LOAD 0.9
 # define SCAN 16
 
+const static size_t UCS4_SIZE = sizeof(Py_UCS4);
 
 typedef enum KeysArrayType{
     KAT_LIST = 0, // must be falsy
@@ -790,8 +791,8 @@ lookup_hash_unicode(
     Py_ssize_t table_pos = hash & mask;
 
     PyArrayObject *a = (PyArrayObject *)self->keys;
-    // REVIEW: is this a new descr reference?
-    Py_ssize_t dt_size = PyArray_DESCR(a)->elsize / sizeof(Py_UCS4);
+    Py_ssize_t dt_size = PyArray_DESCR(a)->elsize / UCS4_SIZE;
+    Py_ssize_t cmp_bytes = Py_MIN(key_size, dt_size) * UCS4_SIZE;
 
     Py_hash_t h = 0;
     Py_UCS4* p_start = NULL;
@@ -808,7 +809,7 @@ lookup_hash_unicode(
             }
             p_start = (Py_UCS4*)PyArray_GETPTR1(a, table[table_pos].keys_pos);
             // memcmp returns 0 on match
-            if (!memcmp(p_start, key, Py_MIN(key_size, dt_size))) {
+            if (!memcmp(p_start, key, cmp_bytes)) {
                 return table_pos;
             }
             table_pos++;
@@ -833,6 +834,7 @@ lookup_hash_string(
 
     PyArrayObject *a = (PyArrayObject *)self->keys;
     Py_ssize_t dt_size = PyArray_DESCR(a)->elsize / sizeof(char);
+    Py_ssize_t cmp_bytes = Py_MIN(key_size, dt_size);
 
     Py_hash_t h = 0;
     char* p_start = NULL;
@@ -849,7 +851,7 @@ lookup_hash_string(
             }
             p_start = (char*)PyArray_GETPTR1(a, table[table_pos].keys_pos);
             // memcmp returns 0 on match
-            if (!memcmp(p_start, key, Py_MIN(key_size, dt_size))) {
+            if (!memcmp(p_start, key, cmp_bytes)) {
                 return table_pos;
             }
             table_pos++;
@@ -1110,7 +1112,7 @@ lookup_unicode(FAMObject *self, PyObject* key) {
         return -1;
     }
     PyArrayObject *a = (PyArrayObject *)self->keys;
-    Py_ssize_t dt_size = PyArray_DESCR(a)->elsize / sizeof(Py_UCS4);
+    Py_ssize_t dt_size = PyArray_DESCR(a)->elsize / UCS4_SIZE;
     // if the key_size is greater than the dtype size of the array, we know there cannot be a match
     Py_ssize_t k_size = PyUnicode_GetLength(key);
     if (k_size > dt_size) {
@@ -1435,8 +1437,8 @@ copy_to_new(PyTypeObject *cls, FAMObject *self, FAMObject *new)
     new->key_buffer = NULL;
     if (new->keys_array_type == KAT_UNICODE) {
         PyArrayObject *a = (PyArrayObject *)new->keys;
-        Py_ssize_t dt_size = PyArray_DESCR(a)->elsize / sizeof(Py_UCS4);
-        new->key_buffer = (Py_UCS4*)PyMem_Malloc((dt_size+1) * sizeof(Py_UCS4));
+        Py_ssize_t dt_size = PyArray_DESCR(a)->elsize / UCS4_SIZE;
+        new->key_buffer = (Py_UCS4*)PyMem_Malloc((dt_size+1) * UCS4_SIZE);
     }
 
     Py_ssize_t table_size_alloc = new->table_size + SCAN - 1;
@@ -1922,8 +1924,8 @@ fam_init(PyObject *self, PyObject *args, PyObject *kwargs)
                 break;
             case KAT_UNICODE: {
                 // Over allocate buffer by 1 so there is room for null at end. This buffer is only used in lookup();
-                Py_ssize_t dt_size = PyArray_DESCR(a)->elsize / sizeof(Py_UCS4);
-                fam->key_buffer = (Py_UCS4*)PyMem_Malloc((dt_size+1) * sizeof(Py_UCS4));
+                Py_ssize_t dt_size = PyArray_DESCR(a)->elsize / UCS4_SIZE;
+                fam->key_buffer = (Py_UCS4*)PyMem_Malloc((dt_size+1) * UCS4_SIZE);
                 INSERT_FLEXIBLE(Py_UCS4, insert_unicode, ucs4_get_end_p);
                 break;
             }

doc/articles/npy-opt.py

@@ -5,9 +5,9 @@
 from typing import NamedTuple
 from itertools import repeat
 
-import automap
-from automap import AutoMap
-from automap import FrozenAutoMap
+import arraymap
+from arraymap import AutoMap
+from arraymap import FrozenAutoMap
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -386,7 +386,7 @@ def get_array(size: int) -> np.ndarray:
 def get_versions() -> str:
     import platform
 
-    return f"OS: {platform.system()} / AutoMap / NumPy: {np.__version__}\n"
+    return f"OS: {platform.system()} / ArrayMap: {arraymap.__version__} / NumPy: {np.__version__}\n"
 
 
 CLS_FF = (
@@ -402,7 +402,7 @@ def get_versions() -> str:
 FF_ORDER = [f.NAME for f in sorted(CLS_FF, key=lambda ff: ff.SORT)]
 
 # -------------------------------------------------------------------------------
-NUMBER = 50
+NUMBER = 2
 
 from itertools import product
 
@@ -476,7 +476,7 @@ def plot_performance(frame, suffix: str = ""):
     fig.text(0.05, 0.96, f"AutoMap {suffix.title()}: {NUMBER} Iterations", fontsize=10)
     fig.text(0.05, 0.90, get_versions(), fontsize=6)
 
-    fp = f"/tmp/automap-{suffix}.png"
+    fp = f"/tmp/arraymap-{suffix}.png"
     plt.subplots_adjust(
         left=0.075,
         bottom=0.05,

test/test_unit.py

@@ -136,6 +136,12 @@ def test_fam_constructor_array_unicode_b():
         assert k in fam
 
 
+def test_fam_constructor_array_unicode_c():
+    a1 = np.array(("z0Ct", "z0DS", "z0E9"))
+    a1.flags.writeable = False
+    fam = FrozenAutoMap(a1)
+
+
 def test_fam_copy_array_unicode_a():
     a1 = np.array(("a", "ccc", "bb"))
     a1.flags.writeable = False