From 34f16c5c0f699e8fe4d39a24162183350e9e481f Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Wed, 14 Oct 2026 11:13:38 +0000 Subject: [PATCH] [cpyrt] Deduce a template instantiation per annotated Python callable An implicit template call with an annotated Python function deduced the template argument from the annotations only the first time. A later call with a differently annotated callable reused that first instantiation, silently wrapping the new callable with the old signature (e.g. truncating a float argument to int). The signature hash behind the memoized dispatch only mixes in the Python type, which all functions share, and the function pointer converter wrapped any callable regardless of its annotations. Mix the annotation values of Python functions and bound methods into HashSignature. Give the function pointer converter the type it converts to and have it, in the strict round, accept an annotated callable only if the annotations declare that function type: the declared signature is resolved to a type once per spelling and compared through CppInterOp, so the spelling need not match the parameter's. A mismatch defers the callable to the implicit round, as for other implicit conversions. The std::function converter skips the check, as a std::function takes any compatible callable. A None annotation now reads as void. Fixes #151 --- src/cpyrt/CPPOverload.cxx | 32 ++++++++ src/cpyrt/CPPOverload.h | 7 ++ src/cpyrt/Converters.cxx | 48 ++++++++++-- src/cpyrt/DeclareConverters.h | 10 ++- src/cpyrt/Utility.cxx | 139 ++++++++++++++++++---------------- src/cpyrt/Utility.h | 10 +++ test/test_templates.py | 101 +++++++++++++++++++++++- 7 files changed, 273 insertions(+), 74 deletions(-) diff --git a/src/cpyrt/CPPOverload.cxx b/src/cpyrt/CPPOverload.cxx index 96016205..baab60ae 100644 --- a/src/cpyrt/CPPOverload.cxx +++ b/src/cpyrt/CPPOverload.cxx @@ -901,6 +901,38 @@ static PyMethodDef mp_methods[] = { } // unnamed namespace +//= cpyrt signature hashing =============================================== +uint64_t HashAnnotations(PyObject* pyfunc) { + // Hash the values of the __annotations__ of a Python callable in order, as + // they declare the C++ function type it converts to (see AnnotatedSignature). + PyObject* annot = PyObject_GetAttr(pyfunc, PyStrings::gAnnotations); + if (!annot) { + PyErr_Clear(); + return 0; + } + + uint64_t hash = 0; + if (PyDict_Check(annot)) { + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(annot, &pos, &key, &value)) { + // strings cache their hash, types hash by identity, generic aliases such + // as list[int] by content (a new object on each evaluation) + Py_hash_t h = PyObject_Hash(value); + if (h == -1) { + PyErr_Clear(); + h = (Py_hash_t)(uintptr_t)value; + } + hash += (uint64_t)h; + hash += (hash << 10); + hash ^= (hash >> 6); + } + } + + Py_DECREF(annot); + return hash; +} + //= cpyrt method proxy type =============================================== PyTypeObject CPPOverload_Type = { PyVarObject_HEAD_INIT(&PyType_Type, diff --git a/src/cpyrt/CPPOverload.h b/src/cpyrt/CPPOverload.h index 87019d26..635e8e8d 100644 --- a/src/cpyrt/CPPOverload.h +++ b/src/cpyrt/CPPOverload.h @@ -12,6 +12,9 @@ namespace cppjit::cpyrt { +// hash of the __annotations__ of a Python function or bound method +uint64_t HashAnnotations(PyObject* pyfunc); + // signature hashes are also used by TemplateProxy inline uint64_t HashSignature(cpyrt_PyArgs_t args, size_t nargsf) { // Build a hash from the types of the given python function arguments. @@ -23,6 +26,10 @@ inline uint64_t HashSignature(cpyrt_PyArgs_t args, size_t nargsf) { // the improved overloads for implicit conversions PyObject* pyobj = cpyrt_PyArgs_GET_ITEM(args, i); hash += (uint64_t)Py_TYPE(pyobj); + // all Python functions share one type, but their annotations select the + // overload or template instantiation they convert to + if (PyFunction_Check(pyobj) || PyMethod_Check(pyobj)) + hash += HashAnnotations(pyobj); #if PY_VERSION_HEX >= 0x030e0000 hash += (uint64_t)(PyUnstable_Object_IsUniqueReferencedTemporary(pyobj) ? 1 diff --git a/src/cpyrt/Converters.cxx b/src/cpyrt/Converters.cxx index 58a4aac2..496f5c78 100644 --- a/src/cpyrt/Converters.cxx +++ b/src/cpyrt/Converters.cxx @@ -2990,6 +2990,26 @@ static void* PyFunction_AsCPointer(PyObject* pyobject, return nullptr; } +bool cpyrt::FunctionPointerConverter::AnnotationsMatch( + PyObject* pyobject) const { + // Whether the __annotations__ of a Python function declare the function type + // being converted to; callables without such a declaration match any, as + // does a converter that does not know its type. Template argument deduction + // derives the instantiation from these annotations, so a mismatch means that + // the callable belongs to a different instantiation (or overload). The + // declared signature is compared as a type, resolved once per spelling, so + // typedefs and qualification do not have to match the parameter's. + if (!fFnType || !(PyFunction_Check(pyobject) || PyMethod_Check(pyobject))) + return true; + + const std::string& sig = Utility::AnnotatedSignature(pyobject); + if (sig.empty()) + return true; + + interop::TCppType_t declared = Utility::AnnotatedSignatureType(sig); + return declared && interop::IsSimilarFnTypes(declared, fFnType); +} + bool cpyrt::FunctionPointerConverter::SetArg(PyObject* pyobject, Parameter& para, CallContext* ctxt) { @@ -3000,7 +3020,22 @@ bool cpyrt::FunctionPointerConverter::SetArg(PyObject* pyobject, return true; } - // normal case, get a function pointer + // a Python callable whose annotations declare a different function type is + // wrapped in the implicit round only, as the wrapper converts the arguments + // and the return value + if (ctxt && !AllowImplicit(ctxt) && !AnnotationsMatch(pyobject)) { + if (!NoImplicit(ctxt)) + ctxt->fFlags |= CallContext::kHaveImplicit; + return false; + } + + return ConvertCallable(pyobject, para, ctxt); +} + +bool cpyrt::FunctionPointerConverter::ConvertCallable(PyObject* pyobject, + Parameter& para, + CallContext* ctxt) { + // get a function pointer for any callable, Python ones through a wrapper void* fptr = PyFunction_AsCPointer(pyobject, fRetType, fSignature, fAllowCppInstance); if (fptr) { @@ -3054,8 +3089,10 @@ bool cpyrt::StdFunctionConverter::SetArg(PyObject* pyobject, Parameter& para, PyErr_Clear(); - // else create a wrapper function - if (this->FunctionPointerConverter::SetArg(pyobject, para, ctxt)) { + // else create a wrapper function; a std::function takes any callable whose + // arguments and result convert, so unlike for a function pointer the + // annotations of a Python callable need not spell the signature + if (ConvertCallable(pyobject, para, ctxt)) { // retrieve the wrapper pointer and capture it in a temporary std::function, // then try normal conversion a second time PyObject* func = @@ -3623,7 +3660,8 @@ cppjit::cpyrt::CreateConverter(const std::string& fullType, cdims_t dims) { auto pos2 = resolvedType.rfind(')'); result = new FunctionPointerConverter( resolvedType.substr(0, pos1), - resolvedType.substr(pos1 + sm.length(), pos2 - 1)); + resolvedType.substr(pos1 + sm.length(), pos2 - 1), + interop::GetType(resolvedType, /*enable_slow_lookup=*/true)); } } const std::string failure_msg("Failed to convert type: " + fullType + @@ -3866,7 +3904,7 @@ cppjit::cpyrt::CreateConverter(interop::TCppType_t type, cdims_t dims) { std::string return_type = resolvedTypeStr.substr(0, pos1); result = new FunctionPointerConverter( return_type.erase(return_type.find_last_not_of(" ") + 1), - resolvedTypeStr.substr(pos2 + 2, pos3 - pos2 - 1)); + resolvedTypeStr.substr(pos2 + 2, pos3 - pos2 - 1), resolvedType); } else if ((realTypeStr != "std::byte") && (klass || (klass = interop::GetFullScope(realTypeStr)))) { // std::byte is a special enum class used to access raw memory diff --git a/src/cpyrt/DeclareConverters.h b/src/cpyrt/DeclareConverters.h index b1b655fd..81672f9b 100644 --- a/src/cpyrt/DeclareConverters.h +++ b/src/cpyrt/DeclareConverters.h @@ -448,8 +448,9 @@ class STLStringMoveConverter : public STLStringConverter { // function pointers class FunctionPointerConverter : public Converter { public: - FunctionPointerConverter(const std::string& ret, const std::string& sig) - : fRetType(ret), fSignature(sig) {} + FunctionPointerConverter(const std::string& ret, const std::string& sig, + interop::TCppType_t fntype = nullptr) + : fRetType(ret), fSignature(sig), fFnType(fntype) {} public: bool SetArg(PyObject*, Parameter&, CallContext* = nullptr) override; @@ -458,9 +459,14 @@ class FunctionPointerConverter : public Converter { bool HasState() override { return true; } std::string GetFailureMsg() override { return "[FunctionPointerConverter]"; }; +protected: + bool AnnotationsMatch(PyObject*) const; + bool ConvertCallable(PyObject*, Parameter&, CallContext*); + protected: std::string fRetType; std::string fSignature; + interop::TCppType_t fFnType; // function (pointer) type converted to, if known bool fAllowCppInstance = false; }; diff --git a/src/cpyrt/Utility.cxx b/src/cpyrt/Utility.cxx index aba0e5b3..a08f1c7e 100644 --- a/src/cpyrt/Utility.cxx +++ b/src/cpyrt/Utility.cxx @@ -413,6 +413,68 @@ static inline std::string AnnotationAsText(PyObject* pyobj) { return PyUnicode_AsUTF8(pyobj); } +static inline std::string AnnotationAsCppType(PyObject* pyobj, bool scoped) { + if (pyobj == Py_None) + return "void"; + if (cpyrt::CPPScope_Check(pyobj)) { + const std::string& name = cpyrt::Utility::ClassName(pyobj); + return scoped ? full_scope(name) : name; + } + return AnnotationAsText(pyobj); +} + +//---------------------------------------------------------------------------- +std::string cpyrt::Utility::AnnotatedSignature(PyObject* pycallable) { + // Read the signature declared by the __annotations__ of a Python callable: + // a return annotation and at least one argument annotation are needed, the + // argument types are taken in declaration order. + PyObject* annot = PyObject_GetAttr(pycallable, PyStrings::gAnnotations); + if (!annot) { + PyErr_Clear(); + return ""; + } + + std::string sig; + if (PyDict_Check(annot) && 1 < PyDict_Size(annot)) { + PyObject* pyret = PyDict_GetItemString(annot, "return"); + // dict is ordered, with the last value being the return type + PyObject* values = pyret ? PyDict_Values(annot) : nullptr; + if (values) { + sig = AnnotationAsCppType(pyret, /*scoped=*/false) + " (*)("; + for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values) - 1); ++i) { + if (i) + sig += ", "; + sig += AnnotationAsCppType(PyList_GET_ITEM(values, i), /*scoped=*/true); + } + sig += ')'; + Py_DECREF(values); + } else + PyErr_Clear(); + } + + Py_DECREF(annot); + return sig; +} + +//---------------------------------------------------------------------------- +interop::TCppType_t +cpyrt::Utility::AnnotatedSignatureType(const std::string& signature) { + // Resolve a signature spelled by annotations to the function pointer type + // it names. The lookup goes through the interpreter, so its result is kept + // per spelling; a failed lookup is not, as a later declaration may make the + // spelling valid. + static std::unordered_map sTypes; + auto it = sTypes.find(signature); + if (it != sTypes.end()) + return it->second; + + interop::TCppType_t type = + interop::GetType(signature, /*enable_slow_lookup=*/true); + if (type) + sTypes[signature] = type; + return type; +} + static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, cpyrt::Utility::ArgPreference pref, int* pcnt = nullptr) { @@ -511,38 +573,11 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, if (arg && PyCallable_Check(arg)) { // annotated/typed Python function - PyObject* annot = PyObject_GetAttr(arg, PyStrings::gAnnotations); - if (annot) { - if (PyDict_Check(annot) && 1 < PyDict_Size(annot)) { - PyObject* ret = PyDict_GetItemString(annot, "return"); - if (ret) { - // dict is ordered, with the last value being the return type - std::ostringstream tpn; - tpn << (CPPScope_Check(ret) ? ClassName(ret) : AnnotationAsText(ret)) - << " (*)("; - - PyObject* values = PyDict_Values(annot); - for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values) - 1); ++i) { - if (i) - tpn << ", "; - PyObject* item = PyList_GET_ITEM(values, i); - tpn << (CPPScope_Check(item) ? full_scope(ClassName(item)) - : AnnotationAsText(item)); - } - Py_DECREF(values); - - tpn << ')'; - tmpl_name.append(tpn.str()); - - Py_DECREF(annot); - return true; - - } else - PyErr_Clear(); - } - Py_DECREF(annot); - } else - PyErr_Clear(); + const std::string& sig = AnnotatedSignature(arg); + if (!sig.empty()) { + tmpl_name.append(sig); + return true; + } // ctypes function pointer PyObject* argtypes = nullptr; @@ -771,40 +806,12 @@ static bool AddTypeName(std::vector& types, PyObject* tn, } if (arg && PyCallable_Check(arg)) { - PyObject* annot = PyObject_GetAttr(arg, PyStrings::gAnnotations); - if (annot) { - if (PyDict_Check(annot) && 1 < PyDict_Size(annot)) { - PyObject* ret = PyDict_GetItemString(annot, "return"); - if (ret) { - // dict is ordered, with the last value being the return type - std::ostringstream tpn; - tpn << (CPPScope_Check(ret) ? ClassName(ret) : PyUnicode_AsUTF8(ret)) - << " (*)("; - - PyObject* values = PyDict_Values(annot); - for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values) - 1); ++i) { - if (i) - tpn << ", "; - PyObject* item = PyList_GET_ITEM(values, i); - tpn << (CPPScope_Check(item) ? ClassName(item) - : PyUnicode_AsUTF8(item)); - } - Py_DECREF(values); - - tpn << ')'; - // tmpl_name.append(tpn.str()); - // FIXME: find a way to add it to types - throw std::runtime_error( - "This path is not yet implemented (AddTypeName) \n"); - - return true; - - } else - PyErr_Clear(); - } - Py_DECREF(annot); - } else - PyErr_Clear(); + // annotated/typed Python function + if (!AnnotatedSignature(arg).empty()) { + // FIXME: find a way to add the function pointer type to types + throw std::runtime_error( + "This path is not yet implemented (AddTypeName) \n"); + } PyObject* tpName = PyObject_GetAttr(arg, PyStrings::gCppName); if (tpName) { diff --git a/src/cpyrt/Utility.h b/src/cpyrt/Utility.h index acd6936c..c9c15ba7 100644 --- a/src/cpyrt/Utility.h +++ b/src/cpyrt/Utility.h @@ -57,6 +57,16 @@ inline PyObject* CT2CppName(PyObject* pytc, const char* cpd, bool allow_voidp) { return nullptr; } +// function pointer type declared by the __annotations__ of a Python callable, +// in C++ spelling ("ret (*)(args)": the annotations are taken as C++ type +// names, Python types by their name, None as void); empty if they do not +// declare a full signature +std::string AnnotatedSignature(PyObject* pycallable); + +// the type spelled by such a signature, as resolved by the interpreter and +// cached per spelling; nullptr if the spelling is not a type it knows +interop::TCppType_t AnnotatedSignatureType(const std::string& signature); + // helper for generating callbacks void ConstructCallbackPreamble(const std::string& retType, const std::vector& argtypes, diff --git a/test/test_templates.py b/test/test_templates.py index 93189de3..74df11f2 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -1453,7 +1453,106 @@ def test39_monkey_patching_template_proxy(self): assert a.m([1, 2, 3]) assert not a.m(42) - def test40_instantiation_failure_error_message(self): + def test40_annotated_callables_select_instantiation(self): + """Annotated Python callables each deduce their own instantiation""" + + import cppjit + + cppjit.cppdef("""\ + #include + namespace AnnotatedCallables { + template + auto call(F f) { return f(2.5); } + + int which(int (*f)(int)) { return 1000 + f(2); } + int which(int (*f)(float)) { return 2000 + f(2.5f); } + using float_t = float; + + double via_std_function(std::function f) { return f(3); } + template + double via_std_function_t(std::function f, T x) { return f(x); } + + void run(void (*f)(int), int x) { f(x); } + + struct Holder { + Holder(std::function f) : value(f(4)) {} + double value; + }; + double via_holder(const Holder& h) { return h.value; } + }""") + + ns = cppjit.gbl.AnnotatedCallables + + def A(x: int) -> int: + return x * 10 + + def B(x: float) -> int: + return int(x * 2) + + def C(x: float) -> float: + return x * 2.0 + + def D(x): + return x + + class K: + def m(self, x: float) -> float: + return x * 3 + + assert ns.call(A) == 20 + + # an unannotated callable is wrapped by the known instantiation + assert ns.call(D) == 2 + + # the first instantiation (from A) used to be reused for B, which was + # then called with its argument truncated to int; repeat to also cover + # the memoized dispatch + for _ in range(2): + assert ns.call(A) == 20 + assert ns.call(B) == 5 + assert ns.call(C) == 5.0 + assert ns.call(K().m) == 7.5 + + assert ns.call["int (*)(float)"](B) == 5 + + # non-template overloads likewise go by the annotations + for _ in range(2): + assert ns.which(A) == 1020 + assert ns.which(B) == 2005 + + # which are compared as types, not by their spelling + def B2(x: "AnnotatedCallables::float_t") -> int: + return int(x * 2) + + assert ns.which(B2) == 2005 + + # std::function takes any callable whose arguments and result convert + def F(x: int) -> float: + return x * 1.5 + + assert ns.via_std_function(F) == 4.5 + assert ns.via_std_function(A) == 30.0 + + # also when selecting among known instantiations, which disallows + # implicit conversions + assert ns.via_std_function_t["int"](F, 2) == 3.0 + assert ns.via_std_function_t(F, 2) == 3.0 + assert ns.via_std_function_t(A, 2) == 20.0 + + # or when converting implicitly through a constructor + assert ns.via_holder(F) == 6.0 + assert ns.via_holder(A) == 40.0 + + # a None annotation declares a void return + seen = [] + + def G(x: int) -> None: + seen.append(x) + + ns.run(G, 7) + assert seen == [7] + + def test41_instantiation_failure_error_message(self): """Rejected instantiation names the template, not garbage""" import cppjit