[cppyy] Treat long and long long buffer formats as interchangeable

bindings/pyroot/cppjit/src/cpyrt/Converters.cxx

@@ -457,7 +457,7 @@ static inline bool CArraySetArg(PyObject* pyobject, cpyrt::Parameter& para,
   if (cpyrt::LowLevelView_Check(pyobject)) {
     auto llview = ((cpyrt::LowLevelView*)pyobject);
     if (llview->fBufInfo.itemsize != size ||
-        !strchr(llview->fBufInfo.format, tc)) {
+        !cpyrt::Utility::BufferFormatCompatible(tc, llview->fBufInfo.format)) {
       PyErr_Format(PyExc_TypeError,
                    "could not convert argument to buffer or nullptr");
       return false;
@@ -1789,7 +1789,7 @@ bool cpyrt::StdSpanConverter::SetArg(PyObject* pyobject, Parameter& para,
   memset(&fBufinfo, 0, sizeof(Py_buffer));
 
   if (PyObject_GetBuffer(pyobject, &fBufinfo, PyBUF_FORMAT) == 0) {
-    if (!strchr(fBufinfo.format, typecode)) {
+    if (!Utility::BufferFormatCompatible(typecode, fBufinfo.format)) {
       PyErr_Format(PyExc_TypeError,
                    "buffer has incompatible type: expected '%c' for C++ type "
                    "'%s', but got format '%s'",
@@ -1880,7 +1880,8 @@ bool ToArrayFromBuffer(PyObject* owner, void* address, PyObject* ctxt,
         convOk = true;                                                         \
       } else if (LowLevelView_Check(pyobject) &&                               \
                  ((LowLevelView*)pyobject)->fBufInfo.ndim == 2 &&              \
-                 strchr(((LowLevelView*)pyobject)->fBufInfo.format, code)) {   \
+                 Utility::BufferFormatCompatible(                              \
+                     code, ((LowLevelView*)pyobject)->fBufInfo.format)) {      \
         para.fValue.fVoidp = ((LowLevelView*)pyobject)->get_buf();             \
         para.fTypeCode = 'p';                                                  \
         convOk = true;                                                         \
@@ -1907,7 +1908,8 @@ bool ToArrayFromBuffer(PyObject* owner, void* address, PyObject* ctxt,
           convOk = true;                                                       \
         }                                                                      \
       } else if (LowLevelView_Check(pyobject) &&                               \
-                 strchr(((LowLevelView*)pyobject)->fBufInfo.format, code)) {   \
+                 Utility::BufferFormatCompatible(                              \
+                     code, ((LowLevelView*)pyobject)->fBufInfo.format)) {      \
         para.fValue.fVoidp = ((LowLevelView*)pyobject)->get_buf();             \
         para.fTypeCode = 'p';                                                  \
         convOk = true;                                                         \

bindings/pyroot/cppjit/src/cpyrt/Utility.cxx

@@ -1201,6 +1201,139 @@ std::unordered_map<std::string, char> const& cpyrt::Utility::TypecodeMap() {
   return typecodeMap;
 }
 
+namespace {
+
+// classification of PEP 3118 format characters
+enum class EBufferKind {
+  kSInt,
+  kUInt,
+  kFloat,
+  kComplex,
+  kChar,
+  kWChar,
+  kBool,
+  kString,
+  kVoidPtr,
+  kUnknown
+};
+
+struct BufferElem {
+  EBufferKind kind;
+  std::size_t size;
+  char code; // the underlying format character
+};
+
+BufferElem ClassifyFormatChar(char c) {
+  switch (c) {
+  case 'b':
+    return {EBufferKind::kSInt, sizeof(char), c};
+  case 'h':
+    return {EBufferKind::kSInt, sizeof(short), c};
+  case 'i':
+    return {EBufferKind::kSInt, sizeof(int), c};
+  case 'l':
+    return {EBufferKind::kSInt, sizeof(long), c};
+  case 'q':
+    return {EBufferKind::kSInt, sizeof(long long), c};
+  case 'n':
+    return {EBufferKind::kSInt, sizeof(Py_ssize_t), c};
+  case 'B':
+    return {EBufferKind::kUInt, sizeof(unsigned char), c};
+  case 'H':
+    return {EBufferKind::kUInt, sizeof(unsigned short), c};
+  case 'I':
+    return {EBufferKind::kUInt, sizeof(unsigned int), c};
+  case 'L':
+    return {EBufferKind::kUInt, sizeof(unsigned long), c};
+  case 'Q':
+    return {EBufferKind::kUInt, sizeof(unsigned long long), c};
+  case 'N':
+    return {EBufferKind::kUInt, sizeof(std::size_t), c};
+  case 'e':
+    return {EBufferKind::kFloat, 2, c}; // IEEE 754 half
+  case 'f':
+    return {EBufferKind::kFloat, sizeof(float), c};
+  case 'd':
+    return {EBufferKind::kFloat, sizeof(double), c};
+  case 'g':
+    return {EBufferKind::kFloat, sizeof(long double), c};
+  case '?':
+    return {EBufferKind::kBool, sizeof(bool), c};
+  case 'c':
+    return {EBufferKind::kChar, sizeof(char), c};
+  case 'u':
+    return {EBufferKind::kWChar, sizeof(wchar_t), c};
+  case 'w':
+    return {EBufferKind::kWChar, sizeof(char32_t), c}; // UCS-4
+  case 's':
+  case 'p':
+    return {EBufferKind::kString, 0, c};
+  case 'P':
+    return {EBufferKind::kVoidPtr, sizeof(void*), c};
+  default:
+    return {EBufferKind::kUnknown, 0, c};
+  }
+}
+
+// reduce a PEP 3118 format string to its element type, skipping an optional
+// byte-order/alignment prefix ("<q") and repeat count ("10q"); fail for
+// formats without a single scalar element (e.g. structured "T{...}" ones)
+bool ParseBufferFormat(const char* fmt, BufferElem& elemOut) {
+  if (!fmt)
+    return false;
+  // strchr also matches on the terminating NUL, so must check for the empty
+  // string first
+  if (*fmt && strchr("@=<>!^", *fmt))
+    ++fmt;
+  while ('0' <= *fmt && *fmt <= '9')
+    ++fmt;
+  bool isComplex = *fmt == 'Z';
+  if (isComplex)
+    ++fmt;
+  if (!*fmt || fmt[1] != '\0')
+    return false;
+
+  BufferElem elem = ClassifyFormatChar(*fmt);
+  if (isComplex) {
+    if (elem.kind != EBufferKind::kFloat)
+      return false;
+    elem = {EBufferKind::kComplex, 2 * elem.size, *fmt};
+  }
+  elemOut = elem;
+  return true;
+}
+
+} // namespace
+
+//----------------------------------------------------------------------------
+bool cpyrt::Utility::BufferFormatCompatible(char tc, const char* fmt) {
+  // the requested type code: a PEP 3118 character, or a cppjit-internal
+  // code ('z' = std::complex<float>, 'Z' = std::complex<double>)
+  BufferElem want;
+  if (tc == 'z')
+    want = {EBufferKind::kComplex, 2 * sizeof(float), tc};
+  else if (tc == 'Z')
+    want = {EBufferKind::kComplex, 2 * sizeof(double), tc};
+  else
+    want = ClassifyFormatChar(tc);
+  if (want.kind == EBufferKind::kUnknown)
+    return false;
+
+  BufferElem got;
+  if (!ParseBufferFormat(fmt, got) || got.kind == EBufferKind::kUnknown)
+    return false;
+
+  // the struct/array modules express bool-ish values as 'signed char' ('b')
+  if (want.kind == EBufferKind::kBool && got.code == 'b')
+    return true;
+
+  // 's' and 'p' are both strings but not interchangeable
+  if (want.kind == EBufferKind::kString)
+    return got.code == want.code;
+
+  return want.kind == got.kind && want.size == got.size;
+}
+
 //----------------------------------------------------------------------------
 Py_ssize_t cpyrt::Utility::GetBuffer(PyObject* pyobject, char tc, int size,
                                      void*& buf, bool check) {
@@ -1231,19 +1364,7 @@ Py_ssize_t cpyrt::Utility::GetBuffer(PyObject* pyobject, char tc, int size,
     Py_buffer bufinfo;
     memset(&bufinfo, 0, sizeof(Py_buffer));
     if (PyObject_GetBuffer(pyobject, &bufinfo, PyBUF_FORMAT) == 0) {
-      if (tc == '*' ||
-          strchr(bufinfo.format, tc)
-          // if `long int` and `int` are the same size (on Windows and 32bit
-          // Linux, for example), `ctypes` isn't too picky about the type
-          // format, so make sure both integer types pass the type check
-          || (sizeof(long int) == sizeof(int) &&
-              ((tc == 'I' && strchr(bufinfo.format, 'L')) ||
-               (tc == 'i' && strchr(bufinfo.format, 'l'))))
-          // complex float is 'Zf' in bufinfo.format, but 'z' in single char
-          || (tc == 'z' && strstr(bufinfo.format, "Zf"))
-          // allow 'signed char' ('b') from array to pass through '?' (bool as
-          // from struct)
-          || (tc == '?' && strchr(bufinfo.format, 'b'))) {
+      if (tc == '*' || BufferFormatCompatible(tc, bufinfo.format)) {
         buf = bufinfo.buf;
 
         if (check && bufinfo.itemsize != size) {

bindings/pyroot/cppjit/src/cpyrt/Utility.h

@@ -73,6 +73,14 @@ bool InitProxy(PyObject* module, PyTypeObject* pytype, const char* name);
 
 std::unordered_map<std::string, char> const& TypecodeMap();
 
+// check whether a PEP 3118 buffer format string describes an element type
+// compatible with the requested type code: same kind (signed/unsigned
+// integer, floating point, complex, ...) and same native size, so e.g.
+// 'long' and 'long long' buffers are interchangeable where they coincide.
+// The type code may be a PEP 3118 character or a cppjit-internal code:
+// 'z' (std::complex<float>) or 'Z' (std::complex<double>).
+bool BufferFormatCompatible(char tc, const char* fmt);
+
 // retrieve the memory buffer from pyobject, return buflength, tc (optional) is
 // python array.array type code, size is type size, buf will point to buffer,
 // and if check is true, some heuristics will be applied to check buffer

bindings/pyroot/cppjit/test/test_datatypes.py

@@ -2777,6 +2777,184 @@
         ns.take_int8(101)
         raises(TypeError, ns.take_int8, "e")
 
+    def test56_long_long_long_array_interchange(self):
+        """Arrays of 64b integers must convert to both 'long' and 'long long'
+        pointer parameters, as buffer producers do not distinguish them and the
+        mapping of int64_t differs across platforms."""
+
+        import array
+        import cppjit
+
+        try:
+            import numpy as np
+        except ImportError:
+            skip('numpy is not installed')
+
+        if array.array('l', []).itemsize != 8:   # 'long' and 'long long' must be same size
+            skip('test assumes 64b long')
+
+        cppjit.cppdef("""\
+        namespace ArrayInterchange {
+        long long sum_ll(const long long* a, int n) {
+            long long s = 0;
+            for (int i = 0; i < n; ++i) s += a[i];
+            return s;
+        }
+        long long sum_l(const long* a, int n) { return sum_ll((const long long*)a, n); }
+        unsigned long long sum_ull(const unsigned long long* a, int n) {
+            unsigned long long s = 0;
+            for (int i = 0; i < n; ++i) s += a[i];
+            return s;
+        }
+        unsigned long long sum_ul(const unsigned long* a, int n) { return sum_ull((const unsigned long long*)a, n); }
+        long long sum_i64(const int64_t* a, int n) { return sum_ll((const long long*)a, n); }
+        const long* data_l() { static long l[3] = {0, 1, 2}; return l; }
+        const long long* data_ll() { static long long l[3] = {0, 1, 2}; return l; }
+        }""")
+
+        ns = cppjit.gbl.ArrayInterchange
+        n = 3
+        isum = sum(range(n))
+
+        # numpy int64 is canonicalized to 'l' or 'q' by the platform
+        ia = np.arange(n, dtype=np.int64)
+        ua = np.arange(n, dtype=np.uint64)
+        for func in [ns.sum_ll, ns.sum_l, ns.sum_i64]:
+            assert func(ia, n) == isum
+        for func in [ns.sum_ull, ns.sum_ul]:
+            assert func(ua, n) == isum
+
+        # same through the standard 'array' module, which can express both formats
+        al, aq = array.array('l', range(n)), array.array('q', range(n))
+        assert ns.sum_ll(al, n) == isum and ns.sum_ll(aq, n) == isum
+        assert ns.sum_l(al, n) == isum and ns.sum_l(aq, n) == isum
+
+        # same for low-level views returned to Python with the other type
+        assert ns.sum_ll(ns.data_l(), n) == isum
+        assert ns.sum_l(ns.data_ll(), n) == isum
+
+        # same through std::span, if available (needs C++20 and <span>)
+        if cppjit.evaluate("""#if __cplusplus >= 202002L && __has_include(<span>)
+        1
+        #else
+        0
+        #endif""") == 1:
+            cppjit.cppdef("""\
+            #include <span>
+            namespace ArrayInterchange {
+            long long sum_span_ll(std::span<const long long> a) {
+                long long s = 0;
+                for (auto v : a) s += v;
+                return s;
+            }
+            long long sum_span_l(std::span<const long> a) {
+                return sum_span_ll(std::span<const long long>{reinterpret_cast<const long long*>(a.data()), a.size()});
+            }
+            }""")
+            assert ns.sum_span_ll(ia) == isum and ns.sum_span_l(ia) == isum
+            assert ns.sum_span_ll(al) == isum and ns.sum_span_ll(aq) == isum
+
+    def test57_buffer_format_parsing(self):
+        """Buffer-format matching parses PEP 3118 properly: byte-order
+        prefixes are skipped, the complex marker 'Z' is understood, and
+        elements match on kind (signed/unsigned/float/complex) and native
+        size rather than on the exact format character."""
+
+        import ctypes
+        import cppjit
+
+        try:
+            import numpy as np
+        except ImportError:
+            skip('numpy is not installed')
+
+        cppjit.cppdef("""\
+        #include <complex>
+        namespace BufferFormats {
+        double sum_d(const double* a, int n) {
+            double s = 0.;
+            for (int i = 0; i < n; ++i) s += a[i];
+            return s;
+        }
+        long sum_l(const long* a, int n) {
+            long s = 0;
+            for (int i = 0; i < n; ++i) s += a[i];
+            return s;
+        }
+        int sum_i(const int* a, int n) {
+            int s = 0;
+            for (int i = 0; i < n; ++i) s += a[i];
+            return s;
+        }
+        int count_true(const bool* a, int n) {
+            int s = 0;
+            for (int i = 0; i < n; ++i) if (a[i]) ++s;
+            return s;
+        }
+        std::complex<float> sum_cf(const std::complex<float>* a, int n) {
+            std::complex<float> s;
+            for (int i = 0; i < n; ++i) s += a[i];
+            return s;
+        }
+        const std::complex<float>* data_cf() {
+            static std::complex<float> c[2] = {{1.f, 1.f}, {1.f, -1.f}};
+            return c;
+        }
+        }""")
+
+        ns = cppjit.gbl.BufferFormats
+
+        # ctypes exports explicitly-endian formats
+        assert ns.sum_d((ctypes.c_double * 3)(0.5, 1.5, 1.0), 3) == 3.0
+        if ctypes.sizeof(ctypes.c_long) == 8:
+            assert ns.sum_l((ctypes.c_long * 3)(1, 2, 3), 3) == 6
+            # ctypes exports c_longlong as '<q', so this also crosses the
+            # 'q'/'l' character with equal size
+            assert ns.sum_l((ctypes.c_longlong * 3)(1, 2, 3), 3) == 6
+
+        # kind and size both matter: int16 is not int32, float is not int
+        assert ns.sum_i(np.arange(3, dtype=np.int32), 3) == 3
+        raises(TypeError, ns.sum_i, np.arange(3, dtype=np.int16), 3)
+        raises(TypeError, ns.sum_i, np.arange(3, dtype=np.int64), 3)
+        raises(TypeError, ns.sum_i, np.arange(3, dtype=np.float32), 3)
+
+        # bool accepts signed-char buffers (struct/array module convention)
+        assert ns.count_true(np.array([1, 0, 1], dtype=np.int8), 3) == 2
+
+        # complex buffers ('Zf'/'Zd') only match complex parameters of the
+        # same size
+        s = ns.sum_cf(np.array([1+1j, 1-1j], dtype=np.complex64), 2)
+        assert s.real == 2.0 and s.imag == 0.0
+        raises(TypeError, ns.sum_d, np.array([1+1j, 1], dtype=np.complex128), 2)
+
+        # a complex view returned from C++ round-trips back into complex
+        # pointer parameters
+        sv = ns.sum_cf(ns.data_cf(), 2)
+        assert sv.real == 2.0 and sv.imag == 0.0
+
+        # same-size integer kinds do not cross the signedness boundary
+        raises(TypeError, ns.sum_l, np.arange(3, dtype=np.uint64), 3)
+
+        # std::span performs no item-size check, so the format check alone
+        # must reject complex128 for span<const double>
+        if cppjit.evaluate("""#if __cplusplus >= 202002L && __has_include(<span>)
+        1
+        #else
+        0
+        #endif""") == 1:
+            cppjit.cppdef("""\
+            #include <span>
+            namespace BufferFormats {
+            double sum_span_d(std::span<const double> a) {
+                double s = 0.;
+                for (auto v : a) s += v;
+                return s;
+            }
+            }""")
+            assert ns.sum_span_d(np.arange(3, dtype=np.float64)) == 3.0
+            raises(TypeError, ns.sum_span_d,
+                   np.array([1+2j, 3+4j], dtype=np.complex128))
+
 
 class TestANONENUM:
     def test01_anonymous_enum_repeated_access(self):