[interop] Make the cling work with -fmodules

CMakeLists.txt

@@ -11,7 +11,7 @@ include(GNUInstallDirs)
 # Perhaps this should permanently be OFF and users can build their own CppInterOp if they want to run the tests?
 option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF)
 set(CPPINTEROP_GIT_REPOSITORY "https://github.com/compiler-research/CppInterOp.git" CACHE STRING "")
-set(CPPINTEROP_GIT_TAG "c8126249dd18363000b89333b34313ca2a83fb82" CACHE STRING "")
+set(CPPINTEROP_GIT_TAG "c57d253e7cba94cc0976a9e36547a960f0fd2f83" CACHE STRING "")
 set(CPPINTEROP_SOURCE_DIR "" CACHE PATH
     "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source")
 
@@ -76,6 +76,7 @@ target_compile_definitions(cppjit PRIVATE
     CPPINTEROP_INCLUDE_DIR="${CPPJIT_INTEROP_RUNTIME_INCLUDES}"
     CPPJIT_CLANG_MAJOR="${CPPJIT_INTEROP_CLANG_MAJOR}"
     CPPJIT_CLANG_INCLUDE_DIR="${CPPJIT_INTEROP_CLANG_DIR}"
+    $<$<BOOL:${CPPJIT_INTEROP_CLING_INCLUDE_DIR}>:CPPJIT_CLING_INCLUDE_DIR="${CPPJIT_INTEROP_CLING_INCLUDE_DIR}">
     $<$<BOOL:${CPPJIT_USE_CLING}>:CPPJIT_USE_CLING>
 )
 

cmake/AddCppInterOp.cmake

@@ -12,6 +12,7 @@
 #   CPPJIT_INTEROP_CLANG_MAJOR        clang major of the interpreter
 #   CPPJIT_INTEROP_CLANG_DIR          resource-dir coordinate; empty = probe
 #   CPPJIT_INTEROP_CLANG_RESOURCE_DIR builtin headers to bundle (install only)
+#   CPPJIT_INTEROP_CLING_INCLUDE_DIR  cling's include dir; empty = repl backend
 include_guard(GLOBAL)
 include(ExternalProject)
 
@@ -43,6 +44,7 @@ function(cppjit_add_cppinterop)
     endif()
 
     set(_clang_resource_dir "")
+    set(_cling_include_dir "")
     if(CppInterOp_FOUND)
         if(CPPINTEROP_SOURCE_DIR)
             message(FATAL_ERROR
@@ -146,6 +148,14 @@ function(cppjit_add_cppinterop)
                 set(_cling_dir "${_prefix}/lib/cmake/cling")
             endif()
             message(STATUS "CppInterOp backend: Cling (found at ${_cling_dir})")
+            # Cling reads its module maps from beside RuntimeUniverse.h, and it
+            # derives that directory from argv[0] -- the host python here. Only
+            # a -I on the interpreter command line reaches setupCxxModules, so
+            # hand the directory to the wrapper.
+            find_package(Cling CONFIG QUIET PATHS "${_cling_dir}" NO_DEFAULT_PATH)
+            if(Cling_FOUND)
+                set(_cling_include_dir "${CLING_INCLUDE_DIRS}")
+            endif()
             list(APPEND _args
                 -DCPPINTEROP_USE_CLING=ON
                 -DCPPINTEROP_USE_REPL=OFF
@@ -249,4 +259,5 @@ function(cppjit_add_cppinterop)
     set(CPPJIT_INTEROP_CLANG_MAJOR "${_clang_major}" PARENT_SCOPE)
     set(CPPJIT_INTEROP_CLANG_DIR "${_clang_dir}" PARENT_SCOPE)
     set(CPPJIT_INTEROP_CLANG_RESOURCE_DIR "${_clang_resource_dir}" PARENT_SCOPE)
+    set(CPPJIT_INTEROP_CLING_INCLUDE_DIR "${_cling_include_dir}" PARENT_SCOPE)
 endfunction()

src/cpyrt/CPPClassMethod.cxx

@@ -50,6 +50,24 @@ PyObject* cpyrt::CPPClassMethod::Call(CPPInstance*& self, cpyrt_PyArgs_t args,
   return this->Execute(nullptr, 0, ctxt);
 }
 
+//----------------------------------------------------------------------------
+Py_ssize_t cpyrt::CPPClassMethod::GetDeallocArgShift(bool isBound,
+                                                     PyObject* const* args,
+                                                     Py_ssize_t nargs) {
+  // a static method takes no object argument, so a bound call drops self
+  if (isBound)
+    return 0;
+
+  // Call function above drops a leading instance that the call does not need,
+  // and then args[0] belongs to no parameter. The check below repeats the one
+  // in Call(), so a change to either one belongs in both.
+  if (nargs > 0 && CPPInstance_Check(args[0]) && fArgsRequired <= nargs - 1 &&
+      interop::IsSubclass(((CPPInstance*)args[0])->ObjectIsA(), GetScope()))
+    return -1;
+
+  return 0;
+}
+
 //----------------------------------------------------------------------------
 PyObject* cpyrt::CPPClassMethod::GetTypeName() {
   PyObject* cppname =

src/cpyrt/CPPClassMethod.h

@@ -14,6 +14,11 @@ class CPPClassMethod : public CPPMethod {
   PyObject* GetTypeName() override;
 
 public:
+  Py_ssize_t GetDeallocArgShift(bool isBound, PyObject* const* args,
+                                Py_ssize_t nargs) override;
+  // a static method has no object argument.
+  Py_ssize_t GetDeallocParamShift() override { return 0; }
+
   PyCallable* Clone() override { return new CPPClassMethod(*this); }
   PyObject* Call(CPPInstance*& self, cpyrt_PyArgs_t args, size_t nargsf,
                  PyObject* kwds, CallContext* ctxt = nullptr) override;

src/cpyrt/CPPFunction.h

@@ -15,6 +15,11 @@ class CPPFunction : public CPPMethod {
   PyObject* GetTypeName() override;
 
 public:
+  // A free function takes its object as a plain first parameter, so it has no
+  // implicit this slot. The shift of a bound call stays the one of CPPMethod,
+  // as AdjustSelf() puts self in that first parameter.
+  Py_ssize_t GetDeallocParamShift() override { return 0; }
+
   PyCallable* Clone() override { return new CPPFunction(*this); }
   PyObject* Call(CPPInstance*& self, cpyrt_PyArgs_t args, size_t nargsf,
                  PyObject* kwds, CallContext* ctxt = nullptr) override;

src/cpyrt/CPPInstance.cxx

@@ -17,6 +17,7 @@ using namespace cppjit;
 
 // Standard
 #include <algorithm>
+#include <cstdlib>
 #include <sstream>
 
 //- data _____________________________________________________________________
@@ -227,7 +228,17 @@ void cpyrt::op_dealloc_nofree(CPPInstance* pyobj) {
     if (pyobj->fFlags & CPPInstance::kIsValue) {
       interop::CallDestructor(klass, cppobj);
       interop::Deallocate(klass, cppobj);
-    } else
+    } else if (pyobj->fFlags & CPPInstance::kIsMalloc)
+      std::free(cppobj);
+    else if (pyobj->fFlags & CPPInstance::kIsNoConstruct) {
+      if (pyobj->fFlags & CPPInstance::kIsArrayAlloc)
+        ::operator delete[](cppobj);
+      else
+        ::operator delete(cppobj);
+    } else if (pyobj->fFlags & CPPInstance::kIsArrayAlloc)
+      interop::Destruct(klass, cppobj, 1);
+    // Default case: just kIsOwner set in all of memory-ownership flags
+    else
       interop::Destruct(klass, cppobj);
   }
   cppobj = nullptr;
@@ -938,10 +949,30 @@ static int op_setownership(CPPInstance* pyobj, PyObject* value, void*) {
   return 0;
 }
 
+// Added for testing purposes
+//-----------------------------------------------------------------------------
+static PyObject* op_get_array_alloc(CPPInstance* pyobj, void*) {
+  return PyBool_FromLong((long)(pyobj->fFlags & CPPInstance::kIsArrayAlloc));
+}
+//-----------------------------------------------------------------------------
+static PyObject* op_get_no_construct(CPPInstance* pyobj, void*) {
+  return PyBool_FromLong((long)(pyobj->fFlags & CPPInstance::kIsNoConstruct));
+}
+//-----------------------------------------------------------------------------
+static PyObject* op_get_malloc(CPPInstance* pyobj, void*) {
+  return PyBool_FromLong((long)(pyobj->fFlags & CPPInstance::kIsMalloc));
+}
 //-----------------------------------------------------------------------------
 static PyGetSetDef op_getset[] = {
     {(char*)"__python_owns__", (getter)op_getownership, (setter)op_setownership,
      (char*)"If true, python manages the life time of this object", nullptr},
+    {(char*)"__is_array_alloc__", (getter)op_get_array_alloc, nullptr,
+     (char*)"If true, the object was allocated with new[]/operator new[]",
+     nullptr},
+    {(char*)"__is_no_construct__", (getter)op_get_no_construct, nullptr,
+     (char*)"If true, the memory is raw: no constructor was run", nullptr},
+    {(char*)"__is_malloc__", (getter)op_get_malloc, nullptr,
+     (char*)"If true, the object was allocated with malloc", nullptr},
     {(char*)nullptr, nullptr, nullptr, nullptr, nullptr}};
 
 //= cpyrt type number stubs to allow dynamic overrides =====================

src/cpyrt/CPPInstance.h

@@ -29,20 +29,24 @@ class CPPInstance {
 public:
   enum EFlags {
     kDefault = 0x0000,
-    kNoWrapConv = 0x0001,  // use type as-is (eg. no smart ptr wrap)
-    kIsOwner = 0x0002,     // Python instance owns C++ object/memory
-    kIsExtended = 0x0004,  // has extended data
-    kIsValue = 0x0008,     // was created from a by-value return
-    kIsReference = 0x0010, // represents one indirection
-    kIsArray = 0x0020,     // represents an array of objects
-    kIsSmartPtr = 0x0040,  // is or embeds a smart pointer
-    kIsPtrPtr = 0x0080,    // represents two indirections
-    kIsRValue = 0x0100,    // can be used as an r-value
-    kIsLValue = 0x0200,    // can be used as an l-value
-    kNoMemReg = 0x0400,    // do not register with memory regulator
-    kIsRegulated = 0x0800, // is registered with memory regulator
-    kIsActual = 0x1000,    // has been downcasted to actual type
-    kHasLifeLine = 0x2000, // has a life line set
+    kNoWrapConv = 0x0001,   // use type as-is (eg. no smart ptr wrap)
+    kIsOwner = 0x0002,      // Python instance owns C++ object/memory
+    kIsExtended = 0x0004,   // has extended data
+    kIsValue = 0x0008,      // was created from a by-value return
+    kIsReference = 0x0010,  // represents one indirection
+    kIsArray = 0x0020,      // represents an array of objects
+    kIsSmartPtr = 0x0040,   // is or embeds a smart pointer
+    kIsPtrPtr = 0x0080,     // represents two indirections
+    kIsRValue = 0x0100,     // can be used as an r-value
+    kIsLValue = 0x0200,     // can be used as an l-value
+    kNoMemReg = 0x0400,     // do not register with memory regulator
+    kIsRegulated = 0x0800,  // is registered with memory regulator
+    kIsActual = 0x1000,     // has been downcasted to actual type
+    kHasLifeLine = 0x2000,  // has a life line set
+    kIsArrayAlloc = 0x4000, // represents a heap allocated array of objects
+    kIsNoConstruct =
+        0x8000, // represents constructor is not called in the allocation
+    kIsMalloc = 0x10000, // is allocated with malloc
   };
 
 public: // public, as the python C-API works with C structs

src/cpyrt/CPPMethod.cxx

@@ -85,6 +85,8 @@ inline void cpyrt::CPPMethod::Copy_(const CPPMethod& /* other */) {
   fExecutor = nullptr;
   fArgIndices = nullptr;
   fArgsRequired = -1;
+  fAllocType.reset();
+  fDeallocType.reset();
 }
 
 //----------------------------------------------------------------------------
@@ -703,6 +705,37 @@ PyObject* cpyrt::CPPMethod::GetArgDefault(int iarg, bool silent) {
 
 bool cpyrt::CPPMethod::IsConst() { return interop::IsConstMethod(GetMethod()); }
 
+//----------------------------------------------------------------------------
+interop::AllocType cpyrt::CPPMethod::GetAllocBehaviour() {
+  if (fAllocType.has_value())
+    return *fAllocType;
+  interop::TCppMethod_t method = GetMethod();
+  if (!method) {
+    fAllocType = interop::AllocType::None;
+    return interop::AllocType::None;
+  }
+  if (!IsPointerType(interop::GetMethodReturnType(method))) {
+    fAllocType = interop::AllocType::None;
+    return interop::AllocType::None;
+  }
+  interop::AllocType attrResult = interop::IsAllocator(method);
+  fAllocType = attrResult;
+  return attrResult;
+}
+
+//----------------------------------------------------------------------------
+interop::DeallocType cpyrt::CPPMethod::GetDeallocBehaviour(size_t index) {
+  if (!fDeallocType) {
+    fDeallocType.emplace();
+    if (interop::TCppMethod_t method = GetMethod())
+      interop::IsDeallocator(method, *fDeallocType);
+  }
+  if (index < fDeallocType->size()) {
+    return (*fDeallocType)[index];
+  }
+  return interop::DeallocType::Opaque;
+}
+
 //----------------------------------------------------------------------------
 PyObject* cpyrt::CPPMethod::GetScopeProxy() {
   // Get or build the scope of this method.

src/cpyrt/CPPMethod.h

@@ -5,6 +5,7 @@
 #include "PyCallable.h"
 
 // Standard
+#include <optional>
 #include <string>
 #include <unordered_map>
 #include <vector>
@@ -62,6 +63,17 @@ class CPPMethod : public PyCallable {
   PyObject* GetCoVarNames() override;
   PyObject* GetArgDefault(int iarg, bool silent = true) override;
   bool IsConst() override;
+  interop::AllocType GetAllocBehaviour() override;
+  interop::DeallocType GetDeallocBehaviour(size_t index) override;
+  // A bound call passes self on its own, so args[0] lines up with the second
+  // slot. A direct call goes through the vector call, which skips the
+  // descriptor, and then self sits in args[0] itself.
+  Py_ssize_t GetDeallocArgShift(bool isBound, PyObject* const* /* args */,
+                                Py_ssize_t /* nargs */) override {
+    return isBound ? 1 : 0;
+  }
+  // the first slot belongs to the implicit object argument of the method
+  Py_ssize_t GetDeallocParamShift() override { return 1; }
 
   PyObject* GetScopeProxy() override;
   interop::TCppFuncAddr_t GetFunctionAddress() override;
@@ -121,6 +133,8 @@ class CPPMethod : public PyCallable {
 protected:
   // cached value that doubles as initialized flag (uninitialized if -1)
   int fArgsRequired;
+  std::optional<interop::AllocType> fAllocType;
+  std::optional<std::vector<interop::DeallocType>> fDeallocType;
 };
 
 } // namespace cppjit::cpyrt