[cpyrt] Deduce a template instantiation per annotated Python callable

src/cpyrt/CPPOverload.cxx

@@ -901,6 +901,38 @@ static PyMethodDef mp_methods[] = {
 
 } // unnamed namespace
 
+//= cpyrt signature hashing ===============================================
+uint64_t HashAnnotations(PyObject* pyfunc) {
+  // Hash the values of the __annotations__ of a Python callable in order, as
+  // they declare the C++ function type it converts to (see AnnotatedSignature).
+  PyObject* annot = PyObject_GetAttr(pyfunc, PyStrings::gAnnotations);
+  if (!annot) {
+    PyErr_Clear();
+    return 0;
+  }
+
+  uint64_t hash = 0;
+  if (PyDict_Check(annot)) {
+    Py_ssize_t pos = 0;
+    PyObject *key, *value;
+    while (PyDict_Next(annot, &pos, &key, &value)) {
+      // strings cache their hash, types hash by identity, generic aliases such
+      // as list[int] by content (a new object on each evaluation)
+      Py_hash_t h = PyObject_Hash(value);
+      if (h == -1) {
+        PyErr_Clear();
+        h = (Py_hash_t)(uintptr_t)value;
+      }
+      hash += (uint64_t)h;
+      hash += (hash << 10);
+      hash ^= (hash >> 6);
+    }
+  }
+
+  Py_DECREF(annot);
+  return hash;
+}
+
 //= cpyrt method proxy type ===============================================
 PyTypeObject CPPOverload_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type,

src/cpyrt/CPPOverload.h

@@ -12,6 +12,9 @@
 
 namespace cppjit::cpyrt {
 
+// hash of the __annotations__ of a Python function or bound method
+uint64_t HashAnnotations(PyObject* pyfunc);
+
 // signature hashes are also used by TemplateProxy
 inline uint64_t HashSignature(cpyrt_PyArgs_t args, size_t nargsf) {
   // Build a hash from the types of the given python function arguments.
@@ -23,6 +26,10 @@ inline uint64_t HashSignature(cpyrt_PyArgs_t args, size_t nargsf) {
     // the improved overloads for implicit conversions
     PyObject* pyobj = cpyrt_PyArgs_GET_ITEM(args, i);
     hash += (uint64_t)Py_TYPE(pyobj);
+    // all Python functions share one type, but their annotations select the
+    // overload or template instantiation they convert to
+    if (PyFunction_Check(pyobj) || PyMethod_Check(pyobj))
+      hash += HashAnnotations(pyobj);
 #if PY_VERSION_HEX >= 0x030e0000
     hash +=
         (uint64_t)(PyUnstable_Object_IsUniqueReferencedTemporary(pyobj) ? 1

src/cpyrt/Converters.cxx

@@ -2990,6 +2990,26 @@ static void* PyFunction_AsCPointer(PyObject* pyobject,
   return nullptr;
 }
 
+bool cpyrt::FunctionPointerConverter::AnnotationsMatch(
+    PyObject* pyobject) const {
+  // Whether the __annotations__ of a Python function declare the function type
+  // being converted to; callables without such a declaration match any, as
+  // does a converter that does not know its type. Template argument deduction
+  // derives the instantiation from these annotations, so a mismatch means that
+  // the callable belongs to a different instantiation (or overload). The
+  // declared signature is compared as a type, resolved once per spelling, so
+  // typedefs and qualification do not have to match the parameter's.
+  if (!fFnType || !(PyFunction_Check(pyobject) || PyMethod_Check(pyobject)))
+    return true;
+
+  const std::string& sig = Utility::AnnotatedSignature(pyobject);
+  if (sig.empty())
+    return true;
+
+  interop::TCppType_t declared = Utility::AnnotatedSignatureType(sig);
+  return declared && interop::IsSimilarFnTypes(declared, fFnType);
+}
+
 bool cpyrt::FunctionPointerConverter::SetArg(PyObject* pyobject,
                                              Parameter& para,
                                              CallContext* ctxt) {
@@ -3000,7 +3020,22 @@ bool cpyrt::FunctionPointerConverter::SetArg(PyObject* pyobject,
     return true;
   }
 
-  // normal case, get a function pointer
+  // a Python callable whose annotations declare a different function type is
+  // wrapped in the implicit round only, as the wrapper converts the arguments
+  // and the return value
+  if (ctxt && !AllowImplicit(ctxt) && !AnnotationsMatch(pyobject)) {
+    if (!NoImplicit(ctxt))
+      ctxt->fFlags |= CallContext::kHaveImplicit;
+    return false;
+  }
+
+  return ConvertCallable(pyobject, para, ctxt);
+}
+
+bool cpyrt::FunctionPointerConverter::ConvertCallable(PyObject* pyobject,
+                                                      Parameter& para,
+                                                      CallContext* ctxt) {
+  // get a function pointer for any callable, Python ones through a wrapper
   void* fptr =
       PyFunction_AsCPointer(pyobject, fRetType, fSignature, fAllowCppInstance);
   if (fptr) {
@@ -3054,8 +3089,10 @@ bool cpyrt::StdFunctionConverter::SetArg(PyObject* pyobject, Parameter& para,
 
   PyErr_Clear();
 
-  // else create a wrapper function
-  if (this->FunctionPointerConverter::SetArg(pyobject, para, ctxt)) {
+  // else create a wrapper function; a std::function takes any callable whose
+  // arguments and result convert, so unlike for a function pointer the
+  // annotations of a Python callable need not spell the signature
+  if (ConvertCallable(pyobject, para, ctxt)) {
     // retrieve the wrapper pointer and capture it in a temporary std::function,
     // then try normal conversion a second time
     PyObject* func =
@@ -3623,7 +3660,8 @@ cppjit::cpyrt::CreateConverter(const std::string& fullType, cdims_t dims) {
       auto pos2 = resolvedType.rfind(')');
       result = new FunctionPointerConverter(
           resolvedType.substr(0, pos1),
-          resolvedType.substr(pos1 + sm.length(), pos2 - 1));
+          resolvedType.substr(pos1 + sm.length(), pos2 - 1),
+          interop::GetType(resolvedType, /*enable_slow_lookup=*/true));
     }
   }
   const std::string failure_msg("Failed to convert type: " + fullType +
@@ -3866,7 +3904,7 @@ cppjit::cpyrt::CreateConverter(interop::TCppType_t type, cdims_t dims) {
     std::string return_type = resolvedTypeStr.substr(0, pos1);
     result = new FunctionPointerConverter(
         return_type.erase(return_type.find_last_not_of(" ") + 1),
-        resolvedTypeStr.substr(pos2 + 2, pos3 - pos2 - 1));
+        resolvedTypeStr.substr(pos2 + 2, pos3 - pos2 - 1), resolvedType);
   } else if ((realTypeStr != "std::byte") &&
              (klass || (klass = interop::GetFullScope(realTypeStr)))) {
     // std::byte is a special enum class used to access raw memory

src/cpyrt/DeclareConverters.h

@@ -448,8 +448,9 @@ class STLStringMoveConverter : public STLStringConverter {
 // function pointers
 class FunctionPointerConverter : public Converter {
 public:
-  FunctionPointerConverter(const std::string& ret, const std::string& sig)
-      : fRetType(ret), fSignature(sig) {}
+  FunctionPointerConverter(const std::string& ret, const std::string& sig,
+                           interop::TCppType_t fntype = nullptr)
+      : fRetType(ret), fSignature(sig), fFnType(fntype) {}
 
 public:
   bool SetArg(PyObject*, Parameter&, CallContext* = nullptr) override;
@@ -458,9 +459,14 @@ class FunctionPointerConverter : public Converter {
   bool HasState() override { return true; }
   std::string GetFailureMsg() override { return "[FunctionPointerConverter]"; };
 
+protected:
+  bool AnnotationsMatch(PyObject*) const;
+  bool ConvertCallable(PyObject*, Parameter&, CallContext*);
+
 protected:
   std::string fRetType;
   std::string fSignature;
+  interop::TCppType_t fFnType; // function (pointer) type converted to, if known
   bool fAllowCppInstance = false;
 };
 

src/cpyrt/Utility.cxx

@@ -413,6 +413,68 @@ static inline std::string AnnotationAsText(PyObject* pyobj) {
   return PyUnicode_AsUTF8(pyobj);
 }
 
+static inline std::string AnnotationAsCppType(PyObject* pyobj, bool scoped) {
+  if (pyobj == Py_None)
+    return "void";
+  if (cpyrt::CPPScope_Check(pyobj)) {
+    const std::string& name = cpyrt::Utility::ClassName(pyobj);
+    return scoped ? full_scope(name) : name;
+  }
+  return AnnotationAsText(pyobj);
+}
+
+//----------------------------------------------------------------------------
+std::string cpyrt::Utility::AnnotatedSignature(PyObject* pycallable) {
+  // Read the signature declared by the __annotations__ of a Python callable:
+  // a return annotation and at least one argument annotation are needed, the
+  // argument types are taken in declaration order.
+  PyObject* annot = PyObject_GetAttr(pycallable, PyStrings::gAnnotations);
+  if (!annot) {
+    PyErr_Clear();
+    return "";
+  }
+
+  std::string sig;
+  if (PyDict_Check(annot) && 1 < PyDict_Size(annot)) {
+    PyObject* pyret = PyDict_GetItemString(annot, "return");
+    // dict is ordered, with the last value being the return type
+    PyObject* values = pyret ? PyDict_Values(annot) : nullptr;
+    if (values) {
+      sig = AnnotationAsCppType(pyret, /*scoped=*/false) + " (*)(";
+      for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values) - 1); ++i) {
+        if (i)
+          sig += ", ";
+        sig += AnnotationAsCppType(PyList_GET_ITEM(values, i), /*scoped=*/true);
+      }
+      sig += ')';
+      Py_DECREF(values);
+    } else
+      PyErr_Clear();
+  }
+
+  Py_DECREF(annot);
+  return sig;
+}
+
+//----------------------------------------------------------------------------
+interop::TCppType_t
+cpyrt::Utility::AnnotatedSignatureType(const std::string& signature) {
+  // Resolve a signature spelled by annotations to the function pointer type
+  // it names. The lookup goes through the interpreter, so its result is kept
+  // per spelling; a failed lookup is not, as a later declaration may make the
+  // spelling valid.
+  static std::unordered_map<std::string, interop::TCppType_t> sTypes;
+  auto it = sTypes.find(signature);
+  if (it != sTypes.end())
+    return it->second;
+
+  interop::TCppType_t type =
+      interop::GetType(signature, /*enable_slow_lookup=*/true);
+  if (type)
+    sTypes[signature] = type;
+  return type;
+}
+
 static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg,
                         cpyrt::Utility::ArgPreference pref,
                         int* pcnt = nullptr) {
@@ -511,38 +573,11 @@ static bool AddTypeName(std::string& tmpl_name, PyObject* tn, PyObject* arg,
 
   if (arg && PyCallable_Check(arg)) {
     // annotated/typed Python function
-    PyObject* annot = PyObject_GetAttr(arg, PyStrings::gAnnotations);
-    if (annot) {
-      if (PyDict_Check(annot) && 1 < PyDict_Size(annot)) {
-        PyObject* ret = PyDict_GetItemString(annot, "return");
-        if (ret) {
-          // dict is ordered, with the last value being the return type
-          std::ostringstream tpn;
-          tpn << (CPPScope_Check(ret) ? ClassName(ret) : AnnotationAsText(ret))
-              << " (*)(";
-
-          PyObject* values = PyDict_Values(annot);
-          for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values) - 1); ++i) {
-            if (i)
-              tpn << ", ";
-            PyObject* item = PyList_GET_ITEM(values, i);
-            tpn << (CPPScope_Check(item) ? full_scope(ClassName(item))
-                                         : AnnotationAsText(item));
-          }
-          Py_DECREF(values);
-
-          tpn << ')';
-          tmpl_name.append(tpn.str());
-
-          Py_DECREF(annot);
-          return true;
-
-        } else
-          PyErr_Clear();
-      }
-      Py_DECREF(annot);
-    } else
-      PyErr_Clear();
+    const std::string& sig = AnnotatedSignature(arg);
+    if (!sig.empty()) {
+      tmpl_name.append(sig);
+      return true;
+    }
 
     // ctypes function pointer
     PyObject* argtypes = nullptr;
@@ -771,40 +806,12 @@ static bool AddTypeName(std::vector<Cpp::TemplateArgInfo>& types, PyObject* tn,
   }
 
   if (arg && PyCallable_Check(arg)) {
-    PyObject* annot = PyObject_GetAttr(arg, PyStrings::gAnnotations);
-    if (annot) {
-      if (PyDict_Check(annot) && 1 < PyDict_Size(annot)) {
-        PyObject* ret = PyDict_GetItemString(annot, "return");
-        if (ret) {
-          // dict is ordered, with the last value being the return type
-          std::ostringstream tpn;
-          tpn << (CPPScope_Check(ret) ? ClassName(ret) : PyUnicode_AsUTF8(ret))
-              << " (*)(";
-
-          PyObject* values = PyDict_Values(annot);
-          for (Py_ssize_t i = 0; i < (PyList_GET_SIZE(values) - 1); ++i) {
-            if (i)
-              tpn << ", ";
-            PyObject* item = PyList_GET_ITEM(values, i);
-            tpn << (CPPScope_Check(item) ? ClassName(item)
-                                         : PyUnicode_AsUTF8(item));
-          }
-          Py_DECREF(values);
-
-          tpn << ')';
-          // tmpl_name.append(tpn.str());
-          // FIXME: find a way to add it to types
-          throw std::runtime_error(
-              "This path is not yet implemented (AddTypeName) \n");
-
-          return true;
-
-        } else
-          PyErr_Clear();
-      }
-      Py_DECREF(annot);
-    } else
-      PyErr_Clear();
+    // annotated/typed Python function
+    if (!AnnotatedSignature(arg).empty()) {
+      // FIXME: find a way to add the function pointer type to types
+      throw std::runtime_error(
+          "This path is not yet implemented (AddTypeName) \n");
+    }
 
     PyObject* tpName = PyObject_GetAttr(arg, PyStrings::gCppName);
     if (tpName) {

src/cpyrt/Utility.h

@@ -57,6 +57,16 @@ inline PyObject* CT2CppName(PyObject* pytc, const char* cpd, bool allow_voidp) {
   return nullptr;
 }
 
+// function pointer type declared by the __annotations__ of a Python callable,
+// in C++ spelling ("ret (*)(args)": the annotations are taken as C++ type
+// names, Python types by their name, None as void); empty if they do not
+// declare a full signature
+std::string AnnotatedSignature(PyObject* pycallable);
+
+// the type spelled by such a signature, as resolved by the interpreter and
+// cached per spelling; nullptr if the spelling is not a type it knows
+interop::TCppType_t AnnotatedSignatureType(const std::string& signature);
+
 // helper for generating callbacks
 void ConstructCallbackPreamble(const std::string& retType,
                                const std::vector<std::string>& argtypes,