diff --git a/src/cpyrt/CPPOverload.cxx b/src/cpyrt/CPPOverload.cxx index 96016205..baab60ae 100644 --- a/src/cpyrt/CPPOverload.cxx +++ b/src/cpyrt/CPPOverload.cxx @@ -901,6 +901,38 @@ static PyMethodDef mp_methods[] = { } // unnamed namespace +//= cpyrt signature hashing =============================================== +uint64_t HashAnnotations(PyObject* pyfunc) { + // Hash the values of the __annotations__ of a Python callable in order, as + // they declare the C++ function type it converts to (see AnnotatedSignature). + PyObject* annot = PyObject_GetAttr(pyfunc, PyStrings::gAnnotations); + if (!annot) { + PyErr_Clear(); + return 0; + } + + uint64_t hash = 0; + if (PyDict_Check(annot)) { + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(annot, &pos, &key, &value)) { + // strings cache their hash, types hash by identity, generic aliases such + // as list[int] by content (a new object on each evaluation) + Py_hash_t h = PyObject_Hash(value); + if (h == -1) { + PyErr_Clear(); + h = (Py_hash_t)(uintptr_t)value; + } + hash += (uint64_t)h; + hash += (hash << 10); + hash ^= (hash >> 6); + } + } + + Py_DECREF(annot); + return hash; +} + //= cpyrt method proxy type =============================================== PyTypeObject CPPOverload_Type = { PyVarObject_HEAD_INIT(&PyType_Type, diff --git a/src/cpyrt/CPPOverload.h b/src/cpyrt/CPPOverload.h index 87019d26..635e8e8d 100644 --- a/src/cpyrt/CPPOverload.h +++ b/src/cpyrt/CPPOverload.h @@ -12,6 +12,9 @@ namespace cppjit::cpyrt { +// hash of the __annotations__ of a Python function or bound method +uint64_t HashAnnotations(PyObject* pyfunc); + // signature hashes are also used by TemplateProxy inline uint64_t HashSignature(cpyrt_PyArgs_t args, size_t nargsf) { // Build a hash from the types of the given python function arguments. @@ -23,6 +26,10 @@ inline uint64_t HashSignature(cpyrt_PyArgs_t args, size_t nargsf) { // the improved overloads for implicit conversions PyObject* pyobj = cpyrt_PyArgs_GET_ITEM(args, i); hash += (uint64_t)Py_TYPE(pyobj); + // all Python functions share one type, but their annotations select the + // overload or template instantiation they convert to + if (PyFunction_Check(pyobj) || PyMethod_Check(pyobj)) + hash += HashAnnotations(pyobj); #if PY_VERSION_HEX >= 0x030e0000 hash += (uint64_t)(PyUnstable_Object_IsUniqueReferencedTemporary(pyobj) ? 1 diff --git a/src/cpyrt/Converters.cxx b/src/cpyrt/Converters.cxx index 58a4aac2..496f5c78 100644 --- a/src/cpyrt/Converters.cxx +++ b/src/cpyrt/Converters.cxx @@ -2990,6 +2990,26 @@ static void* PyFunction_AsCPointer(PyObject* pyobject, return nullptr; } +bool cpyrt::FunctionPointerConverter::AnnotationsMatch( + PyObject* pyobject) const { + // Whether the __annotations__ of a Python function declare the function type + // being converted to; callables without such a declaration match any, as + // does a converter that does not know its type. Template argument deduction + // derives the instantiation from these annotations, so a mismatch means that + // the callable belongs to a different instantiation (or overload). The + // declared signature is compared as a type, resolved once per spelling, so + // typedefs and qualification do not have to match the parameter's. + if (!fFnType || !(PyFunction_Check(pyobject) || PyMethod_Check(pyobject))) + return true; + + const std::string& sig = Utility::AnnotatedSignature(pyobject); + if (sig.empty()) + return true; + + interop::TCppType_t declared = Utility::AnnotatedSignatureType(sig); + return declared && interop::IsSimilarFnTypes(declared, fFnType); +} + bool cpyrt::FunctionPointerConverter::SetArg(PyObject* pyobject, Parameter& para, CallContext* ctxt) { @@ -3000,7 +3020,22 @@ bool cpyrt::FunctionPointerConverter::SetArg(PyObject* pyobject, return true; } - // normal case, get a function pointer + // a Python callable whose annotations declare a different function type is + // wrapped in the implicit round only, as the wrapper converts the arguments + // and the return value + if (ctxt && !AllowImplicit(ctxt) && !AnnotationsMatch(pyobject)) { + if (!NoImplicit(ctxt)) + ctxt->fFlags |= CallContext::kHaveImplicit; + return false; + } + + return ConvertCallable(pyobject, para, ctxt); +} + +bool cpyrt::FunctionPointerConverter::ConvertCallable(PyObject* pyobject, + Parameter& para, + CallContext* ctxt) { + // get a function pointer for any callable, Python ones through a wrapper void* fptr = PyFunction_AsCPointer(pyobject, fRetType, fSignature, fAllowCppInstance); if (fptr) { @@ -3054,8 +3089,10 @@ bool cpyrt::StdFunctionConverter::SetArg(PyObject* pyobject, Parameter& para, PyErr_Clear(); - // else create a wrapper function - if (this->FunctionPointerConverter::SetArg(pyobject, para, ctxt)) { + // else create a wrapper function; a std::function takes any callable whose + // arguments and result convert, so unlike for a function pointer the + // annotations of a Python callable need not spell the signature + if (ConvertCallable(pyobject, para, ctxt)) { // retrieve the wrapper pointer and capture it in a temporary std::function, // then try normal conversion a second time PyObject* func = @@ -3623,7 +3660,8 @@ cppjit::cpyrt::CreateConverter(const std::string& fullType, cdims_t dims) { auto pos2 = resolvedType.rfind(')'); result = new FunctionPointerConverter( resolvedType.substr(0, pos1), - resolvedType.substr(pos1 + sm.length(), pos2 - 1)); + resolvedType.substr(pos1 + sm.length(), pos2 - 1), + interop::GetType(resolvedType, /*enable_slow_lookup=*/true)); } } const std::string failure_msg("Failed to convert type: " + fullType + @@ -3866,7 +3904,7 @@ cppjit::cpyrt::CreateConverter(interop::TCppType_t type, cdims_t dims) { std::string return_type = resolvedTypeStr.substr(0, pos1); result = new FunctionPointerConverter( return_type.erase(return_type.find_last_not_of(" ") + 1), - resolvedTypeStr.substr(pos2 + 2, pos3 - pos2 - 1)); + resolvedTypeStr.substr(pos2 + 2, pos3 - pos2 - 1), resolvedType); } else if ((realTypeStr != "std::byte") && (klass || (klass = interop::GetFullScope(realTypeStr)))) { // std::byte is a special enum class used to access raw memory diff --git a/src/cpyrt/DeclareConverters.h b/src/cpyrt/DeclareConverters.h index b1b655fd..81672f9b 100644 --- a/src/cpyrt/DeclareConverters.h +++ b/src/cpyrt/DeclareConverters.h @@ -448,8 +448,9 @@ class STLStringMoveConverter : public STLStringConverter { // function pointers class FunctionPointerConverter : public Converter { public: - FunctionPointerConverter(const std::string& ret, const std::string& sig) - : fRetType(ret), fSignature(sig) {} + FunctionPointerConverter(const std::string& ret, const std::string& sig, + interop::TCppType_t fntype = nullptr) + : fRetType(ret), fSignature(sig), fFnType(fntype) {} public: bool SetArg(PyObject*, Parameter&, CallContext* = nullptr) override; @@ -458,9 +459,14 @@ class FunctionPointerConverter : public Converter { bool HasState() override { return true; } std::string GetFailureMsg() override { return "[FunctionPointerConverter]"; }; +protected: + bool AnnotationsMatch(PyObject*) const; + bool ConvertCallable(PyObject*, Parameter&, CallContext*); + protected: std::string fRetType; std::string fSignature; + interop::TCppType_t fFnType; // function (pointer) type converted to, if known bool fAllowCppInstance = false; }; diff --git a/src/cpyrt/Utility.cxx b/src/cpyrt/Utility.cxx index aba0e5b3..a08f1c7e 100644 --- a/src/cpyrt/Utility.cxx +++ b/src/cpyrt/Utility.cxx @@ -413,6 +413,68 @@ static inline std::string AnnotationAsText(PyObject* pyobj) { return PyUnicode_AsUTF8(pyobj); } +static inline std::string AnnotationAsCppType(PyObject* pyobj, bool scoped) { + if (pyobj == Py_None) + return "void"; + if (cpyrt::CPPScope_Check(pyobj)) { + const std::string& name = cpyrt::Utility::ClassName(pyobj); + return scoped ? full_scope(name) : name; + } + return AnnotationAsText(pyobj); +} + +//---------------------------------------------------------------------------- +std::string cpyrt::Utility::AnnotatedSignature(PyObject* pycallable) { + // Read the signature declared by the __annotations__ of a Python callable: + // a return annotation and at least one argument annotation are needed, the + // argument types are taken in declaration order. + PyObject* annot = PyObject_GetAttr(pycallable, PyStrings::gAnnotations); + if (!annot) { + PyErr_Clear(); + return ""; + } + + std::string sig; + if (PyDict_Check(annot) && 1 < PyDict_Size(annot)) { + PyObject* pyret = PyDict_GetItemString(annot, "return"); + // dict is ordered, with the last value being the return type + PyObject* values = pyret ? PyDict_Values(annot) : nullptr; + if (values) { + sig = AnnotationAsCppType(pyret, /*scoped=*/false) + " (*)("; + for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values) - 1); ++i) { + if (i) + sig += ", "; + sig += AnnotationAsCppType(PyList_GET_ITEM(values, i), /*scoped=*/true); + } + sig += ')'; + Py_DECREF(values); + } else + PyErr_Clear(); + } + + Py_DECREF(annot); + return sig; +} + +//---------------------------------------------------------------------------- +interop::TCppType_t +cpyrt::Utility::AnnotatedSignatureType(const std::string& signature) { + // Resolve a signature spelled by annotations to the function pointer type + // it names. The lookup goes through the interpreter, so its result is kept + // per spelling; a failed lookup is not, as a later declaration may make the + // spelling valid. + static std::unordered_map sTypes; + auto it = sTypes.find(signature); + if (it != sTypes.end()) + return it->second; + + interop::TCppType_t type = + interop::GetType(signature, /*enable_slow_lookup=*/true); + if (type) + sTypes[signature] = type; + return type; +} + static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, cpyrt::Utility::ArgPreference pref, int* pcnt = nullptr) { @@ -511,38 +573,11 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg, if (arg && PyCallable_Check(arg)) { // annotated/typed Python function - PyObject* annot = PyObject_GetAttr(arg, PyStrings::gAnnotations); - if (annot) { - if (PyDict_Check(annot) && 1 < PyDict_Size(annot)) { - PyObject* ret = PyDict_GetItemString(annot, "return"); - if (ret) { - // dict is ordered, with the last value being the return type - std::ostringstream tpn; - tpn << (CPPScope_Check(ret) ? ClassName(ret) : AnnotationAsText(ret)) - << " (*)("; - - PyObject* values = PyDict_Values(annot); - for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values) - 1); ++i) { - if (i) - tpn << ", "; - PyObject* item = PyList_GET_ITEM(values, i); - tpn << (CPPScope_Check(item) ? full_scope(ClassName(item)) - : AnnotationAsText(item)); - } - Py_DECREF(values); - - tpn << ')'; - tmpl_name.append(tpn.str()); - - Py_DECREF(annot); - return true; - - } else - PyErr_Clear(); - } - Py_DECREF(annot); - } else - PyErr_Clear(); + const std::string& sig = AnnotatedSignature(arg); + if (!sig.empty()) { + tmpl_name.append(sig); + return true; + } // ctypes function pointer PyObject* argtypes = nullptr; @@ -771,40 +806,12 @@ static bool AddTypeName(std::vector& types, PyObject* tn, } if (arg && PyCallable_Check(arg)) { - PyObject* annot = PyObject_GetAttr(arg, PyStrings::gAnnotations); - if (annot) { - if (PyDict_Check(annot) && 1 < PyDict_Size(annot)) { - PyObject* ret = PyDict_GetItemString(annot, "return"); - if (ret) { - // dict is ordered, with the last value being the return type - std::ostringstream tpn; - tpn << (CPPScope_Check(ret) ? ClassName(ret) : PyUnicode_AsUTF8(ret)) - << " (*)("; - - PyObject* values = PyDict_Values(annot); - for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values) - 1); ++i) { - if (i) - tpn << ", "; - PyObject* item = PyList_GET_ITEM(values, i); - tpn << (CPPScope_Check(item) ? ClassName(item) - : PyUnicode_AsUTF8(item)); - } - Py_DECREF(values); - - tpn << ')'; - // tmpl_name.append(tpn.str()); - // FIXME: find a way to add it to types - throw std::runtime_error( - "This path is not yet implemented (AddTypeName) \n"); - - return true; - - } else - PyErr_Clear(); - } - Py_DECREF(annot); - } else - PyErr_Clear(); + // annotated/typed Python function + if (!AnnotatedSignature(arg).empty()) { + // FIXME: find a way to add the function pointer type to types + throw std::runtime_error( + "This path is not yet implemented (AddTypeName) \n"); + } PyObject* tpName = PyObject_GetAttr(arg, PyStrings::gCppName); if (tpName) { diff --git a/src/cpyrt/Utility.h b/src/cpyrt/Utility.h index acd6936c..c9c15ba7 100644 --- a/src/cpyrt/Utility.h +++ b/src/cpyrt/Utility.h @@ -57,6 +57,16 @@ inline PyObject* CT2CppName(PyObject* pytc, const char* cpd, bool allow_voidp) { return nullptr; } +// function pointer type declared by the __annotations__ of a Python callable, +// in C++ spelling ("ret (*)(args)": the annotations are taken as C++ type +// names, Python types by their name, None as void); empty if they do not +// declare a full signature +std::string AnnotatedSignature(PyObject* pycallable); + +// the type spelled by such a signature, as resolved by the interpreter and +// cached per spelling; nullptr if the spelling is not a type it knows +interop::TCppType_t AnnotatedSignatureType(const std::string& signature); + // helper for generating callbacks void ConstructCallbackPreamble(const std::string& retType, const std::vector& argtypes, diff --git a/test/test_templates.py b/test/test_templates.py index 93189de3..74df11f2 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -1453,7 +1453,106 @@ def test39_monkey_patching_template_proxy(self): assert a.m([1, 2, 3]) assert not a.m(42) - def test40_instantiation_failure_error_message(self): + def test40_annotated_callables_select_instantiation(self): + """Annotated Python callables each deduce their own instantiation""" + + import cppjit + + cppjit.cppdef("""\ + #include + namespace AnnotatedCallables { + template + auto call(F f) { return f(2.5); } + + int which(int (*f)(int)) { return 1000 + f(2); } + int which(int (*f)(float)) { return 2000 + f(2.5f); } + using float_t = float; + + double via_std_function(std::function f) { return f(3); } + template + double via_std_function_t(std::function f, T x) { return f(x); } + + void run(void (*f)(int), int x) { f(x); } + + struct Holder { + Holder(std::function f) : value(f(4)) {} + double value; + }; + double via_holder(const Holder& h) { return h.value; } + }""") + + ns = cppjit.gbl.AnnotatedCallables + + def A(x: int) -> int: + return x * 10 + + def B(x: float) -> int: + return int(x * 2) + + def C(x: float) -> float: + return x * 2.0 + + def D(x): + return x + + class K: + def m(self, x: float) -> float: + return x * 3 + + assert ns.call(A) == 20 + + # an unannotated callable is wrapped by the known instantiation + assert ns.call(D) == 2 + + # the first instantiation (from A) used to be reused for B, which was + # then called with its argument truncated to int; repeat to also cover + # the memoized dispatch + for _ in range(2): + assert ns.call(A) == 20 + assert ns.call(B) == 5 + assert ns.call(C) == 5.0 + assert ns.call(K().m) == 7.5 + + assert ns.call["int (*)(float)"](B) == 5 + + # non-template overloads likewise go by the annotations + for _ in range(2): + assert ns.which(A) == 1020 + assert ns.which(B) == 2005 + + # which are compared as types, not by their spelling + def B2(x: "AnnotatedCallables::float_t") -> int: + return int(x * 2) + + assert ns.which(B2) == 2005 + + # std::function takes any callable whose arguments and result convert + def F(x: int) -> float: + return x * 1.5 + + assert ns.via_std_function(F) == 4.5 + assert ns.via_std_function(A) == 30.0 + + # also when selecting among known instantiations, which disallows + # implicit conversions + assert ns.via_std_function_t["int"](F, 2) == 3.0 + assert ns.via_std_function_t(F, 2) == 3.0 + assert ns.via_std_function_t(A, 2) == 20.0 + + # or when converting implicitly through a constructor + assert ns.via_holder(F) == 6.0 + assert ns.via_holder(A) == 40.0 + + # a None annotation declares a void return + seen = [] + + def G(x: int) -> None: + seen.append(x) + + ns.run(G, 7) + assert seen == [7] + + def test41_instantiation_failure_error_message(self): """Rejected instantiation names the template, not garbage""" import cppjit