[cppinterop] Upgrade to upstream 656390ee9f5d

interpreter/CppInterOp/CMakeLists.txt

@@ -23,6 +23,9 @@ option(CPPINTEROP_INSTALL "Generate install rules for CppInterOp" ON)
 # would otherwise materialize the SOVERSION namelink as a second full copy.
 option(CPPINTEROP_SHARED_LIBRARY_VERSIONING
   "Attach the LLVM SOVERSION and namelink to shared clangCppInterOp" ON)
+
+set(CPPINTEROP_BAKED_INTERPRETER_ARGS "" CACHE STRING
+  "Space-separated arguments passed at interpreter creation (no quoting: arguments containing spaces are not supported)")
 if(EMSCRIPTEN)
   set(CPPINTEROP_EXTRA_WASM_FLAGS "-fwasm-exceptions" CACHE STRING "Extra flags for wasm")
 endif()

interpreter/CppInterOp/cppinterop-version.tag

@@ -1 +1 @@
-261bebbf25c8f709d2e2490fe6e3f876f6ec5920
+a27f782a9477fd138761377c4b5d4d8316f887dd

interpreter/CppInterOp/include/CppInterOp/CppInterOpTypes.h

@@ -26,6 +26,18 @@
 #endif
 #endif
 
+// Visibility of friend redeclarations of API functions. MSVC treats a friend
+// declaration without the dllimport/dllexport attribute as a
+// different-linkage redeclaration, so it needs CPPINTEROP_API. Elsewhere the
+// visibility comes from the API declaration itself; leaving it off here lets
+// Dispatch.h define these functions with hidden visibility (Clang rejects a
+// visibility that differs from a previous declaration).
+#if defined _WIN32 || defined __CYGWIN__
+#define CPPINTEROP_FRIEND_API CPPINTEROP_API
+#else
+#define CPPINTEROP_FRIEND_API
+#endif
+
 // Cross-platform deprecation attribute. Older Clang versions (Cling)
 // mis-parse C++11 `[[deprecated]]` near the return type, so we use
 // the vendor-specific spelling on each compiler.
@@ -456,8 +468,8 @@ enum class ValueKind : std::uint8_t {
 /// function, constructor or destructor.
 class JitCall {
 public:
-  friend CPPINTEROP_API JitCall MakeFunctionCallable(InterpRef I,
-                                                     ConstFuncRef func);
+  friend CPPINTEROP_FRIEND_API JitCall MakeFunctionCallable(InterpRef I,
+                                                            ConstFuncRef func);
   enum Kind : char {
     kUnknown = 0,
     kGenericCall,
@@ -498,16 +510,13 @@ class JitCall {
       : m_DestructorCall(C), m_Kind(K), m_FD(Dtor) {}
 
   // Trace-hook impls need private m_FD for the function-name lookup.
-  // CPPINTEROP_API matches the X-macro-generated decl in CppInterOpDecl.inc;
-  // MSVC treats a friend decl without the dllimport/dllexport attribute as
-  // a different-linkage redeclaration.
-  friend CPPINTEROP_API void
+  friend CPPINTEROP_FRIEND_API void
   CppInterOpTraceJitCallInvokeImpl(const JitCall*, void* result, void** args,
                                    std::size_t nargs, void* self);
-  friend CPPINTEROP_API void
+  friend CPPINTEROP_FRIEND_API void
   CppInterOpTraceJitCallInvokeDestructorImpl(const JitCall*, void* object,
                                              unsigned long nary, int withFree);
-  friend CPPINTEROP_API void
+  friend CPPINTEROP_FRIEND_API void
   CppInterOpTraceJitCallInvokeReturnImpl(const JitCall*, void* result);
 
   /// Checks if the passed arguments are valid for the given function.

interpreter/CppInterOp/include/CppInterOp/Dispatch.h

@@ -119,10 +119,22 @@ using namespace Cpp;
 // exclusive (#error guard at top) so only one body of Cpp::* exists per TU.
 // CN (CppName) may differ from DN (DispatchName) for overloaded functions,
 // e.g. CN=GetFunctionAddress, DN=GetFunctionAddress_fn.
+//
+// The wrappers have hidden visibility so that the funtion is not exported
+// from the client. Otherwise the dynamic linker could bind libclangCppInterOp's
+// own references to Cpp::*
+#if defined(__GNUC__) && !defined(_WIN32) && !defined(__CYGWIN__)
+#define CPPINTEROP_DISPATCH_HIDDEN __attribute__((__visibility__("hidden")))
+#else
+#define CPPINTEROP_DISPATCH_HIDDEN
+#endif
+
 namespace Cpp {
 
 #define CPPINTEROP_API_FUNC(DN, CN, Ret, DeclArgs, CallArgs, RawTypes)         \
-  inline Ret CN DeclArgs { return ::CppInternal::DispatchRaw::DN CallArgs; }
+  CPPINTEROP_DISPATCH_HIDDEN inline Ret CN DeclArgs {                          \
+    return ::CppInternal::DispatchRaw::DN CallArgs;                            \
+  }
 #include "CppInterOp/CppInterOpAPI.inc"
 
 /// Initialize all CppInterOp API from the dynamically loaded library.

interpreter/CppInterOp/lib/CppInterOp/BakedArgs.h.in

@@ -0,0 +1,5 @@
+// Generated by CMake from BakedArgs.h.in. Do not edit.
+// Carry configure-time values into CppInterOp.cpp through a header instead
+// of -D definitions, so only one escaping layer (the C string literal) ever
+// applies — with -D, the shell/ninja/packaging layers mangle backslashes.
+@_baked_defines@

interpreter/CppInterOp/lib/CppInterOp/CMakeLists.txt

@@ -303,7 +303,54 @@ if(TARGET CppInterOpTableGen)
 endif()
 target_compile_definitions(clangCppInterOp PUBLIC "_CINDEX_LIB_") # workaround for the use of `CINDEX_LINKAGE`
 
+# Where clang keeps its builtin headers. MakeResourcesPath() derives this
+# from LLVM_BINARY_DIR at runtime, which misses wherever clang's headers are
+# installed under a different prefix than LLVM's libraries, leaving a probe
+# for a `clang` on PATH as the only fallback. Ask the clang this build links
+# against instead, while the answer is still knowable. The cache entry is
+# the override; the detected value stays local so it cannot outlive a
+# reconfigure against a different Clang_DIR.
+set(CPPINTEROP_CLANG_RESOURCE_DIR "" CACHE PATH
+  "Clang resource directory compiled into the library; detected when empty")
+
+set(_clang_resource_dir "${CPPINTEROP_CLANG_RESOURCE_DIR}")
+# Auto-detection is skipped for coverage-instrumented builds: a baked-in
+# directory makes MakeResourcesPath() return early at runtime, leaving the
+# runtime fallback paths below it uncovered. An explicit
+# CPPINTEROP_CLANG_RESOURCE_DIR override is always honored.
+if(NOT _clang_resource_dir AND NOT CODE_COVERAGE
+   AND EXISTS "${CLANG_INSTALL_PREFIX}/bin/clang")
+  execute_process(
+    COMMAND "${CLANG_INSTALL_PREFIX}/bin/clang" -print-resource-dir
+    OUTPUT_VARIABLE _detected_resource_dir OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_QUIET)
+  if(EXISTS "${_detected_resource_dir}/include")
+    set(_clang_resource_dir "${_detected_resource_dir}")
+  endif()
+endif()
+message(STATUS "Clang resource directory: ${_clang_resource_dir}")
+
 string(REPLACE ";" "\;" _VER CPPINTEROP_VERSION)
 set_source_files_properties(CppInterOp.cpp PROPERTIES COMPILE_DEFINITIONS
   "LLVM_BINARY_DIR=\"${LLVM_BINARY_DIR}\";CPPINTEROP_VERSION=\"${_VAR}\""
 )
+
+# The values land in C string literals, so escape for exactly that layer.
+# They are delivered through a generated header (see BakedArgs.h.in) instead
+# of -D definitions: with -D, the shell would apply another unescape layer
+# and corrupt literal backslashes on the way to the preprocessor.
+function(_baked_string_define OUT NAME VALUE)
+  string(REPLACE "\\" "\\\\" _v "${VALUE}")
+  string(REPLACE "\"" "\\\"" _v "${_v}")
+  set(${OUT} "${${OUT}}#define ${NAME} \"${_v}\"\n" PARENT_SCOPE)
+endfunction()
+set(_baked_defines "")
+if(CPPINTEROP_BAKED_INTERPRETER_ARGS)
+  _baked_string_define(_baked_defines CPPINTEROP_BAKED_INTERPRETER_ARGS
+    "${CPPINTEROP_BAKED_INTERPRETER_ARGS}")
+endif()
+if(_clang_resource_dir)
+  _baked_string_define(_baked_defines CPPINTEROP_CLANG_RESOURCE_DIR
+    "${_clang_resource_dir}")
+endif()
+configure_file(BakedArgs.h.in ${CMAKE_CURRENT_BINARY_DIR}/BakedArgs.h @ONLY)

interpreter/CppInterOp/lib/CppInterOp/CppInterOp.cpp

@@ -11,6 +11,9 @@
 #include "Unwrap.h"
 #include "CppInterOp/Error.h"
 
+// Generated at configure time; defines CPPINTEROP_BAKED_INTERPRETER_ARGS
+// and CPPINTEROP_CLANG_RESOURCE_DIR when the build baked them in.
+#include "BakedArgs.h"
 #include "Compatibility.h"
 #include "ErrorInternal.h"
 #include "InterpreterInfo.h"
@@ -76,6 +79,7 @@
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/TemplateDeduction.h"
 
+#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
@@ -113,9 +117,9 @@
 #include <deque>
 #include <iostream>
 #include <iterator>
+#include <limits>
 #include <map>
 #include <memory>
-#include <mutex>
 #include <optional>
 #include <set>
 #include <sstream>
@@ -779,6 +783,7 @@ size_t SizeOf(ConstDeclRef DRef) {
 
   if (const auto* RD = dyn_cast<RecordDecl>(unwrap<Decl>(DRef))) {
     ASTContext& Context = RD->getASTContext();
+    compat::SynthesizingCodeRAII RAII(&getInterp());
     const ASTRecordLayout& Layout = Context.getASTRecordLayout(RD);
     return INTEROP_RETURN(Layout.getSize().getQuantity());
   }
@@ -1121,6 +1126,9 @@ static Decl* GetScopeFromType(QualType QT) {
   if (auto* Type = QT.getCanonicalType().getTypePtrOrNull()) {
     Type = Type->getPointeeOrArrayElementType();
     Type = Type->getUnqualifiedDesugaredType();
+    // getDecl() and getAsCXXRecordDecl() walk the redecl chain, which may
+    // deserialize it.
+    compat::SynthesizingCodeRAII RAII(&getInterp());
     if (auto* ET = llvm::dyn_cast<EnumType>(Type))
       return ET->getDecl();
     CXXRecordDecl* CXXRD = Type->getAsCXXRecordDecl();
@@ -1324,7 +1332,7 @@ bool HasReachableUsingDirective(const clang::DeclContext* DC) {
 
 DeclRef GetNamed(const std::string& name, ConstDeclRef parent /*= nullptr*/) {
   INTEROP_TRACE(name, parent);
-  clang::DeclContext* Within = 0;
+  clang::DeclContext* Within = nullptr;
   if (parent) {
     auto* D = unwrap<clang::Decl>(GetUnderlyingScope(parent));
     Within = llvm::dyn_cast<clang::DeclContext>(D);
@@ -1425,6 +1433,9 @@ size_t GetNumBases(ConstDeclRef DRef) {
   INTEROP_TRACE(DRef);
   const auto* D = unwrap<Decl>(DRef);
 
+  // hasDefinition() completes the redecl chain (dataPtr), which may
+  // deserialize it; so does getNumBases() below.
+  compat::SynthesizingCodeRAII RAII(&getInterp());
   if (const auto* CTSD =
           llvm::dyn_cast_or_null<ClassTemplateSpecializationDecl>(D))
     if (!CTSD->hasDefinition())
@@ -1442,6 +1453,7 @@ DeclRef GetBaseClass(ConstDeclRef DRef, size_t ibase) {
   INTEROP_TRACE(DRef, ibase);
   const auto* D = unwrap<Decl>(DRef);
   const auto* CXXRD = llvm::dyn_cast_or_null<CXXRecordDecl>(D);
+  compat::SynthesizingCodeRAII RAII(&getInterp());
   if (!CXXRD || CXXRD->getNumBases() <= ibase)
     return INTEROP_RETURN(nullptr);
 
@@ -3311,6 +3323,8 @@ void GetEnumConstantDatamembers(ConstDeclRef DRef,
   INTEROP_TRACE(DRef, INTEROP_OUT(datamembers), include_enum_class);
   std::vector<DeclRef> EDs;
   GetClassDecls<EnumDecl>(DRef, EDs);
+  // enumerator_begin() loads the enumerators, which may deserialize them.
+  compat::SynthesizingCodeRAII RAII(&getInterp());
   for (DeclRef i : EDs) {
     auto* ED = unwrap<EnumDecl>(i);
 
@@ -3346,6 +3360,7 @@ DeclRef LookupDatamember(const std::string& name, ConstDeclRef parent) {
 bool IsLambdaClass(ConstTypeRef TyRef) {
   INTEROP_TRACE(TyRef);
   QualType QT = QualType::getFromOpaquePtr(TyRef.data);
+  compat::SynthesizingCodeRAII RAII(&getInterp());
   if (auto* CXXRD = QT->getAsCXXRecordDecl()) {
     return INTEROP_RETURN(CXXRD->isLambda());
   }
@@ -3385,6 +3400,56 @@ TypeRef GetVariableType(ConstDeclRef var) {
   return INTEROP_RETURN(nullptr);
 }
 
+// Lay an evaluated constant array initializer out contiguously in
+// ASTContext-owned memory (stable for the interpreter's lifetime) and return
+// its address. Integral element types only; anything else returns 0 so the
+// caller falls back to the regular JIT lookup/codegen path. Cache is the
+// per-interpreter ConstArrayValueStore, so repeated queries return the same
+// address instead of re-evaluating and re-allocating on every call.
+static intptr_t
+MaterializeConstArrayValue(ASTContext& C, const VarDecl* VD, const APValue& Val,
+                           std::map<const VarDecl*, intptr_t>& Cache) {
+  const clang::ArrayType* AT =
+      Val.isArray() ? C.getAsArrayType(VD->getType()) : nullptr;
+  const uint64_t EltBytes =
+      AT ? C.getTypeSizeInChars(AT->getElementType()).getQuantity() : 0;
+  const size_t Count = AT ? Val.getArraySize() : 0;
+  if (!EltBytes || !Count ||
+      Count > std::numeric_limits<uint32_t>::max() / EltBytes)
+    return 0;
+  QualType EltTy = AT->getElementType();
+
+  // The buffer lives in the ASTContext bump allocator for the interpreter's
+  // lifetime, so re-laying out on every call would hand out unstable
+  // addresses (unlike the scalar path, which returns the cached APValue
+  // storage) and grow the allocator unboundedly.
+  auto [It, Inserted] = Cache.try_emplace(VD->getCanonicalDecl(), 0);
+  if (!Inserted)
+    return It->second;
+
+  uint8_t* Buf = static_cast<uint8_t*>(C.Allocate(
+      EltBytes * Count, (unsigned)C.getTypeAlignInChars(EltTy).getQuantity()));
+  for (size_t I = 0; I < Count; ++I) {
+    const APValue* Elt = nullptr;
+    if (I < Val.getArrayInitializedElts())
+      Elt = &Val.getArrayInitializedElt(I);
+    else if (Val.hasArrayFiller())
+      Elt = &Val.getArrayFiller();
+    if (!Elt || !Elt->isInt())
+      return 0; // cached 0: not fully evaluable, fall back to the JIT path
+    const llvm::APInt& Int = Elt->getInt();
+    // The raw layout only works if the APInt fills the element exactly;
+    // padded storage like _BitInt(24) (4-byte storage, 3-byte APInt) would
+    // trip StoreIntToMemory's width assert in assertions-enabled builds.
+    if (Int.getBitWidth() != EltBytes * 8)
+      return 0;
+    llvm::StoreIntToMemory(Int, Buf + (I * EltBytes), (unsigned)EltBytes);
+  }
+  // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+  It->second = reinterpret_cast<intptr_t>(Buf);
+  return It->second;
+}
+
 intptr_t GetVariableOffset(compat::Interpreter& I, Decl* D,
                            CXXRecordDecl* BaseCXXRD) {
   if (!D)
@@ -3461,6 +3526,32 @@ intptr_t GetVariableOffset(compat::Interpreter& I, Decl* D,
     void* address = llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(
         mangledName.c_str());
 
+    // A const variable whose constant initializer is in the AST: serve the
+    // evaluated value before the JIT lookup. This must stay *after* the
+    // search in the loaded binaries above: a symbol that a binary does
+    // export keeps its real address. A missed lookup falls through to the
+    // symbol autoloader, which scans and parses every library on the
+    // search path (over a thousand file opens with ROOT, ~0.5 s), only to
+    // end up in the evaluate fallback below anyway: in-class initialized
+    // static members like TString::kNPOS are exported by no binary. Unlike
+    // that fallback, don't instantiate templates here, so no Sema work is
+    // added for variables the lookup would have found.
+    if (!address) {
+      const VarDecl* InitVD = VD->hasInit() ? VD : VD->getDefinition();
+      if (InitVD && InitVD->hasInit() &&
+          !InitVD->getType().isVolatileQualified() &&
+          (InitVD->isConstexpr() || InitVD->getType().isConstQualified())) {
+        if (const APValue* val = InitVD->evaluateValue()) {
+          if (InitVD->getType()->isIntegralType(C))
+            // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+            return reinterpret_cast<intptr_t>(val->getInt().getRawData());
+          if (intptr_t ArrAddr = MaterializeConstArrayValue(
+                  C, InitVD, *val, getInterpInfo(&I).ConstArrayValueStore))
+            return ArrAddr;
+        }
+      }
+    }
+
     if (!address)
       address = I.getAddressOfGlobal(GD);
     if (!address) {
@@ -5349,6 +5440,13 @@ bool DefineAbsoluteSymbol(compat::Interpreter& I, const char* unmangled_name,
 #endif
 
 static std::string MakeResourcesPath() {
+#ifdef CPPINTEROP_CLANG_RESOURCE_DIR
+  // Recorded at configure time from the clang this library links against;
+  // the LLVM_BINARY_DIR derivation below misses whenever clang's headers
+  // are installed under a different prefix than LLVM's libraries.
+  if (sys::fs::is_directory(CPPINTEROP_CLANG_RESOURCE_DIR))
+    return CPPINTEROP_CLANG_RESOURCE_DIR;
+#endif
   StringRef Dir;
 #ifdef LLVM_BINARY_DIR
   Dir = LLVM_BINARY_DIR;
@@ -5441,13 +5539,37 @@ InterpRef CreateInterpreter(const std::vector<const char*>& Args /*={}*/,
   // copies and move them into the interpreter's InterpreterInfo entry.
   std::vector<std::string> ArgvStorage;
   ArgvStorage.push_back(sys::fs::getMainExecutable(nullptr, nullptr));
+  // Arguments compiled in at configure time, e.g. a packager recording the
+  // toolchain header paths the in-process clang cannot probe for. They
+  // precede the embedder's arguments so they act as defaults: conflicts in
+  // the clang invocation are resolved later-wins, and the embedder's
+  // -resource-dir stays ahead of the baked one (ExtractArgument below).
+  // CPPINTEROP_EXTRA_INTERPRETER_ARGS from the environment follows both.
+  std::vector<const char*> AllArgs;
+#ifdef CPPINTEROP_BAKED_INTERPRETER_ARGS
+  // Storage for the split arguments; must outlive the pointers in AllArgs.
+  std::vector<std::string> BakedStorage;
+  StringRef Baked(CPPINTEROP_BAKED_INTERPRETER_ARGS);
+  while (!Baked.empty()) {
+    StringRef Arg;
+    std::tie(Arg, Baked) = Baked.split(' ');
+    if (!Arg.empty())
+      BakedStorage.push_back(Arg.str());
+  }
+  for (const std::string& Arg : BakedStorage)
+    AllArgs.push_back(Arg.c_str());
+#endif
+  AllArgs.insert(AllArgs.end(), Args.begin(), Args.end());
   // In some systems, CppInterOp cannot manually detect the correct resource.
   // Then the -resource-dir passed by the user is assumed to be the correct
   // location. Prioritising it over detecting it within CppInterOp. Extracting
   // the resource-dir from the arguments is required because we set the
   // necessary library search location explicitly below. Because by default,
   // linker flags are ignored in repl (issue #748)
+  // The embedder's -resource-dir wins over any baked one.
   std::string ResourceDir = ExtractArgument(Args, "-resource-dir");
+  if (ResourceDir.empty())
+    ResourceDir = ExtractArgument(AllArgs, "-resource-dir");
   if (ResourceDir.empty())
     ResourceDir = MakeResourcesPath();
   llvm::Triple T(llvm::sys::getProcessTriple());
@@ -5489,7 +5611,7 @@ InterpRef CreateInterpreter(const std::vector<const char*>& Args /*={}*/,
     }
   }
 #endif
-  ArgvStorage.insert(ArgvStorage.end(), Args.begin(), Args.end());
+  ArgvStorage.insert(ArgvStorage.end(), AllArgs.begin(), AllArgs.end());
   // To keep the Interpreter creation interface between cling and clang-repl
   // to some extent compatible we should put Args and GpuArgs together. On the
   // receiving end we should check for -xcuda to know.