From 620bad06c6e987368eee7f4df702eabd15e2e364 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 6 Oct 2026 08:33:32 +0000 Subject: [PATCH] [cpyrt] Parse buffer formats properly instead of character search numpy and other buffer producers canonicalize the PEP 3118 format char of 64-bit integer dtypes (e.g. int64 is always reported as 'l' on 64-bit platforms), but C++ parameters declared as int64_t resolve to 'long long' on macOS and 'long' on Linux. The exact format-char matching in the array converters therefore rejected valid buffers when the resolved type and the canonicalized buffer format differed, breaking e.g. passing numpy int64 arrays to 'const int64_t*' parameters on macOS. Replace the ad-hoc buffer-format matching with a small PEP 3118 parser in Utility::BufferFormatCompatible(). Formats are reduced to their element type (skipping byte-order prefixes like ctypes' 'fBufInfo.itemsize != size || - !strchr(llview->fBufInfo.format, tc)) { + !cpyrt::Utility::BufferFormatCompatible(tc, llview->fBufInfo.format)) { PyErr_Format(PyExc_TypeError, "could not convert argument to buffer or nullptr"); return false; @@ -1755,7 +1755,7 @@ bool cpyrt::StdSpanConverter::SetArg(PyObject* pyobject, Parameter& para, memset(&fBufinfo, 0, sizeof(Py_buffer)); if (PyObject_GetBuffer(pyobject, &fBufinfo, PyBUF_FORMAT) == 0) { - if (!strchr(fBufinfo.format, typecode)) { + if (!Utility::BufferFormatCompatible(typecode, fBufinfo.format)) { PyErr_Format(PyExc_TypeError, "buffer has incompatible type: expected '%c' for C++ type " "'%s', but got format '%s'", @@ -1846,7 +1846,8 @@ bool ToArrayFromBuffer(PyObject* owner, void* address, PyObject* ctxt, convOk = true; \ } else if (LowLevelView_Check(pyobject) && \ ((LowLevelView*)pyobject)->fBufInfo.ndim == 2 && \ - strchr(((LowLevelView*)pyobject)->fBufInfo.format, code)) { \ + Utility::BufferFormatCompatible( \ + code, ((LowLevelView*)pyobject)->fBufInfo.format)) { \ para.fValue.fVoidp = ((LowLevelView*)pyobject)->get_buf(); \ para.fTypeCode = 'p'; \ convOk = true; \ @@ -1873,7 +1874,8 @@ bool ToArrayFromBuffer(PyObject* owner, void* address, PyObject* ctxt, convOk = true; \ } \ } else if (LowLevelView_Check(pyobject) && \ - strchr(((LowLevelView*)pyobject)->fBufInfo.format, code)) { \ + Utility::BufferFormatCompatible( \ + code, ((LowLevelView*)pyobject)->fBufInfo.format)) { \ para.fValue.fVoidp = ((LowLevelView*)pyobject)->get_buf(); \ para.fTypeCode = 'p'; \ convOk = true; \ diff --git a/src/cpyrt/Utility.cxx b/src/cpyrt/Utility.cxx index aba0e5b3..7575e183 100644 --- a/src/cpyrt/Utility.cxx +++ b/src/cpyrt/Utility.cxx @@ -1179,6 +1179,143 @@ std::unordered_map const& cpyrt::Utility::TypecodeMap() { return typecodeMap; } +namespace { + +// classification of PEP 3118 format characters +enum class EBufferKind { + kSInt, + kUInt, + kFloat, + kComplex, + kChar, + kWChar, + kBool, + kString, + kVoidPtr, + kUnknown +}; + +struct BufferElem { + EBufferKind kind; + std::size_t size; + char code; // the underlying format character +}; + +BufferElem ClassifyFormatChar(char c) { + switch (c) { + case 'b': + return {EBufferKind::kSInt, sizeof(char), c}; + case 'h': + return {EBufferKind::kSInt, sizeof(short), c}; + case 'i': + return {EBufferKind::kSInt, sizeof(int), c}; + case 'l': + return {EBufferKind::kSInt, sizeof(long), c}; + case 'q': + return {EBufferKind::kSInt, sizeof(long long), c}; + case 'n': + return {EBufferKind::kSInt, sizeof(Py_ssize_t), c}; + case 'B': + return {EBufferKind::kUInt, sizeof(unsigned char), c}; + case 'H': + return {EBufferKind::kUInt, sizeof(unsigned short), c}; + case 'I': + return {EBufferKind::kUInt, sizeof(unsigned int), c}; + case 'L': + return {EBufferKind::kUInt, sizeof(unsigned long), c}; + case 'Q': + return {EBufferKind::kUInt, sizeof(unsigned long long), c}; + case 'N': + return {EBufferKind::kUInt, sizeof(std::size_t), c}; + case 'e': + return {EBufferKind::kFloat, 2, c}; // IEEE 754 half + case 'f': + return {EBufferKind::kFloat, sizeof(float), c}; + case 'd': + return {EBufferKind::kFloat, sizeof(double), c}; + case 'g': + return {EBufferKind::kFloat, sizeof(long double), c}; + case '?': + return {EBufferKind::kBool, sizeof(bool), c}; + case 'c': + return {EBufferKind::kChar, sizeof(char), c}; + case 'u': + return {EBufferKind::kWChar, sizeof(wchar_t), c}; + case 'w': + return {EBufferKind::kWChar, sizeof(char32_t), c}; // UCS-4 + case 's': + case 'p': + return {EBufferKind::kString, 0, c}; + case 'P': + return {EBufferKind::kVoidPtr, sizeof(void*), c}; + default: + return {EBufferKind::kUnknown, 0, c}; + } +} + +// reduce a PEP 3118 format string to its element type, skipping an optional +// byte-order/alignment prefix ("!^", *fmt)) + ++fmt; + while ('0' <= *fmt && *fmt <= '9') + ++fmt; + bool isComplex = *fmt == 'Z'; + if (isComplex) + ++fmt; + if (!*fmt || fmt[1] != '\0') + return false; + + BufferElem elem = ClassifyFormatChar(*fmt); + if (isComplex) { + if (elem.kind != EBufferKind::kFloat) + return false; + elem = {EBufferKind::kComplex, 2 * elem.size, *fmt}; + } + elemOut = elem; + return true; +} + +} // namespace + +//---------------------------------------------------------------------------- +bool cpyrt::Utility::BufferFormatCompatible(char tc, const char* fmt) { + // the requested type code: a PEP 3118 character, or a cppjit-internal + // code ('z' = std::complex, 'Z' = std::complex) + BufferElem want; + if (tc == 'z') + want = {EBufferKind::kComplex, 2 * sizeof(float), tc}; + else if (tc == 'Z') + want = {EBufferKind::kComplex, 2 * sizeof(double), tc}; + else + want = ClassifyFormatChar(tc); + if (want.kind == EBufferKind::kUnknown) + return false; + + BufferElem got; + if (!ParseBufferFormat(fmt, got) || got.kind == EBufferKind::kUnknown) + return false; + + // the struct/array modules express bool-ish values as 'signed char' ('b') + if (want.kind == EBufferKind::kBool && got.code == 'b') + return true; + + // 's' and 'p' are both strings but not interchangeable + if (want.kind == EBufferKind::kString) + return got.code == want.code; + + return want.kind == got.kind && want.size == got.size; +} + //---------------------------------------------------------------------------- Py_ssize_t cpyrt::Utility::GetBuffer(PyObject* pyobject, char tc, int size, void*& buf, bool check) { @@ -1209,19 +1346,7 @@ Py_ssize_t cpyrt::Utility::GetBuffer(PyObject* pyobject, char tc, int size, Py_buffer bufinfo; memset(&bufinfo, 0, sizeof(Py_buffer)); if (PyObject_GetBuffer(pyobject, &bufinfo, PyBUF_FORMAT) == 0) { - if (tc == '*' || - strchr(bufinfo.format, tc) - // if `long int` and `int` are the same size (on Windows and 32bit - // Linux, for example), `ctypes` isn't too picky about the type - // format, so make sure both integer types pass the type check - || (sizeof(long int) == sizeof(int) && - ((tc == 'I' && strchr(bufinfo.format, 'L')) || - (tc == 'i' && strchr(bufinfo.format, 'l')))) - // complex float is 'Zf' in bufinfo.format, but 'z' in single char - || (tc == 'z' && strstr(bufinfo.format, "Zf")) - // allow 'signed char' ('b') from array to pass through '?' (bool as - // from struct) - || (tc == '?' && strchr(bufinfo.format, 'b'))) { + if (tc == '*' || BufferFormatCompatible(tc, bufinfo.format)) { buf = bufinfo.buf; if (check && bufinfo.itemsize != size) { diff --git a/src/cpyrt/Utility.h b/src/cpyrt/Utility.h index acd6936c..28186c31 100644 --- a/src/cpyrt/Utility.h +++ b/src/cpyrt/Utility.h @@ -73,6 +73,14 @@ bool InitProxy(PyObject* module, PyTypeObject* pytype, const char* name); std::unordered_map const& TypecodeMap(); +// check whether a PEP 3118 buffer format string describes an element type +// compatible with the requested type code: same kind (signed/unsigned +// integer, floating point, complex, ...) and same native size, so e.g. +// 'long' and 'long long' buffers are interchangeable where they coincide. +// The type code may be a PEP 3118 character or a cppjit-internal code: +// 'z' (std::complex) or 'Z' (std::complex). +bool BufferFormatCompatible(char tc, const char* fmt); + // retrieve the memory buffer from pyobject, return buflength, tc (optional) is // python array.array type code, size is type size, buf will point to buffer, // and if check is true, some heuristics will be applied to check buffer diff --git a/test/test_datatypes.py b/test/test_datatypes.py index e6071fd4..cb25ec73 100644 --- a/test/test_datatypes.py +++ b/test/test_datatypes.py @@ -2799,6 +2799,196 @@ def test56_function_pointer_return_multiword_types(self): # its signature, which must keep "const char" and "unsigned int" assert cppjit.gbl.ns56.get()("abc", 4) == 7 + def test57_long_long_long_array_interchange(self): + """Arrays of 64b integers must convert to both 'long' and 'long long' + pointer parameters, as buffer producers do not distinguish them and the + mapping of int64_t differs across platforms.""" + + import array + + import cppjit + + try: + import numpy as np + except ImportError: + skip("numpy is not installed") + + # 'long' and 'long long' must be same size + if array.array("l", []).itemsize != 8: + skip("test assumes 64b long") + + cppjit.cppdef("""\ + namespace ArrayInterchange { + long long sum_ll(const long long* a, int n) { + long long s = 0; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + long long sum_l(const long* a, int n) { return sum_ll((const long long*)a, n); } + unsigned long long sum_ull(const unsigned long long* a, int n) { + unsigned long long s = 0; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + unsigned long long sum_ul(const unsigned long* a, int n) { return sum_ull((const unsigned long long*)a, n); } + long long sum_i64(const int64_t* a, int n) { return sum_ll((const long long*)a, n); } + const long* data_l() { static long l[3] = {0, 1, 2}; return l; } + const long long* data_ll() { static long long l[3] = {0, 1, 2}; return l; } + }""") + + ns = cppjit.gbl.ArrayInterchange + n = 3 + isum = sum(range(n)) + + # numpy int64 is canonicalized to 'l' or 'q' by the platform + ia = np.arange(n, dtype=np.int64) + ua = np.arange(n, dtype=np.uint64) + for func in [ns.sum_ll, ns.sum_l, ns.sum_i64]: + assert func(ia, n) == isum + for func in [ns.sum_ull, ns.sum_ul]: + assert func(ua, n) == isum + + # same through the standard 'array' module, which can express both formats + al, aq = array.array("l", range(n)), array.array("q", range(n)) + assert ns.sum_ll(al, n) == isum and ns.sum_ll(aq, n) == isum + assert ns.sum_l(al, n) == isum and ns.sum_l(aq, n) == isum + + # same for low-level views returned to Python with the other type + assert ns.sum_ll(ns.data_l(), n) == isum + assert ns.sum_l(ns.data_ll(), n) == isum + + # same through std::span, if available (needs C++20 and ) + if ( + cppjit.evaluate("""#if __cplusplus >= 202002L && __has_include() + 1 + #else + 0 + #endif""") + == 1 + ): + cppjit.cppdef("""\ + #include + namespace ArrayInterchange { + long long sum_span_ll(std::span a) { + long long s = 0; + for (auto v : a) s += v; + return s; + } + long long sum_span_l(std::span a) { + return sum_span_ll(std::span{reinterpret_cast(a.data()), a.size()}); + } + }""") + assert ns.sum_span_ll(ia) == isum and ns.sum_span_l(ia) == isum + assert ns.sum_span_ll(al) == isum and ns.sum_span_ll(aq) == isum + + def test58_buffer_format_parsing(self): + """Buffer-format matching parses PEP 3118 properly: byte-order + prefixes are skipped, the complex marker 'Z' is understood, and + elements match on kind (signed/unsigned/float/complex) and native + size rather than on the exact format character.""" + + import ctypes + + import cppjit + + try: + import numpy as np + except ImportError: + skip("numpy is not installed") + + cppjit.cppdef("""\ + #include + namespace BufferFormats { + double sum_d(const double* a, int n) { + double s = 0.; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + long sum_l(const long* a, int n) { + long s = 0; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + int sum_i(const int* a, int n) { + int s = 0; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + int count_true(const bool* a, int n) { + int s = 0; + for (int i = 0; i < n; ++i) if (a[i]) ++s; + return s; + } + std::complex sum_cf(const std::complex* a, int n) { + std::complex s; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + const std::complex* data_cf() { + static std::complex c[2] = {{1.f, 1.f}, {1.f, -1.f}}; + return c; + } + }""") + + ns = cppjit.gbl.BufferFormats + + # ctypes exports explicitly-endian formats + assert ns.sum_d((ctypes.c_double * 3)(0.5, 1.5, 1.0), 3) == 3.0 + if ctypes.sizeof(ctypes.c_long) == 8: + assert ns.sum_l((ctypes.c_long * 3)(1, 2, 3), 3) == 6 + # ctypes exports c_longlong as ' + if ( + cppjit.evaluate("""#if __cplusplus >= 202002L && __has_include() + 1 + #else + 0 + #endif""") + == 1 + ): + cppjit.cppdef("""\ + #include + namespace BufferFormats { + double sum_span_d(std::span a) { + double s = 0.; + for (auto v : a) s += v; + return s; + } + }""") + assert ns.sum_span_d(np.arange(3, dtype=np.float64)) == 3.0 + raises( + TypeError, + ns.sum_span_d, + np.array([1 + 2j, 3 + 4j], dtype=np.complex128), + ) + class TestANONENUM: def test01_anonymous_enum_repeated_access(self):