diff --git a/bindings/pyroot/cppjit/src/cpyrt/API.cxx b/bindings/pyroot/cppjit/src/cpyrt/API.cxx index b0da815ec12b4..82b405c3f795f 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/API.cxx +++ b/bindings/pyroot/cppjit/src/cpyrt/API.cxx @@ -11,12 +11,9 @@ using namespace cppjit; #include "CPPOverload.h" #include "CPPScope.h" #include "ProxyWrappers.h" -#include "PyStrings.h" #include "cpyrt/DispatchPtr.h" // Standard -#include -#include #include //______________________________________________________________________________ @@ -25,11 +22,6 @@ using namespace cppjit; // // Access to cppjit Python objects from Cling and C++: allows conversion for // instances and type checking for scopes, instances, etc. -// Adds a few convenience functions to call Python from Cling and expose Python -// classes to Cling for use in inheritance etc. - -//- data --------------------------------------------------------------------- -static PyObject* gMainDict = nullptr; namespace cppjit::cpyrt { extern PyObject* gThisModule; @@ -39,27 +31,17 @@ extern PyObject* gThisModule; namespace { static bool Initialize() { - // Private initialization method: setup the python interpreter and load the - // cppjit module. + // Private initialization method: load the cppjit module. Expects the Python + // interpreter to be already initialized. static bool isInitialized = false; if (isInitialized) return true; - if (!Py_IsInitialized()) { - // this happens if Cling comes in first - PyConfig config; - PyConfig_InitPythonConfig(&config); - PyConfig_SetString(&config, &config.program_name, L"cppjit"); - Py_InitializeFromConfig(&config); - - // try again to see if the interpreter is initialized - if (!Py_IsInitialized()) { - // give up ... - std::cerr << "Error: python has not been initialized; returning." - << std::endl; - return false; - } - } + // cppjit is a Python extension library, not an embedding library: the + // Python interpreter must be initialized by the embedding application + // (e.g. TPython) before any cpyrt API function is called. + if (!Py_IsInitialized()) + return false; // Importing the extension module is what runs the cpyrt initialization // that sets gThisModule. @@ -70,17 +52,6 @@ static bool Initialize() { Py_DECREF(cppjitmod); } - if (!gMainDict) { - cpyrt::PythonGILRAII python_gil_raii; - // retrieve the main dictionary - gMainDict = - PyModule_GetDict(PyImport_AddModule(const_cast("__main__"))); - // The gMainDict is borrowed, i.e. we are not calling Py_INCREF(gMainDict). - // Like this, we avoid unexpectedly affecting how long __main__ is kept - // alive. The gMainDict is only used in Exec(), ExecScript(), and Eval(), - // which should not be called after __main__ is garbage collected anyway. - } - // declare success ... isInitialized = true; return true; @@ -306,172 +277,12 @@ void cpyrt::Instance_SetReduceMethod(PyCFunction reduceMethod) { CPPInstance::ReduceMethod() = reduceMethod; } -//- access to the python interpreter ---------------------------------------- -bool cpyrt::Import(const std::string& mod_name) { - // Import the named python module and create Cling equivalents for its - // classes. - if (!Initialize()) - return false; - - PythonGILRAII python_gil_raii; - - PyObject* mod = PyImport_ImportModule(mod_name.c_str()); - if (!mod) { - PyErr_Print(); - return false; - } - - // allow finding to prevent creation of a python proxy for the C++ proxy - Py_INCREF(mod); - PyModule_AddObject(gThisModule, mod_name.c_str(), mod); - - // force creation of the module as a namespace - // TODO: the following is broken (and should live in cppjit_interop.cxx) - // TClass::GetClass(mod_name, true); - - PyObject* dct = PyModule_GetDict(mod); - - // create Cling classes for all new python classes - PyObject* values = PyDict_Values(dct); - for (int i = 0; i < PyList_GET_SIZE(values); ++i) { - PyObject* value = PyList_GET_ITEM(values, i); - Py_INCREF(value); - - // collect classes - if (PyClass_Check(value) || PyObject_HasAttr(value, PyStrings::gBases)) { - // get full class name (including module) - PyObject* pyClName = PyObject_GetAttr(value, PyStrings::gName); - if (PyErr_Occurred()) - PyErr_Clear(); - - // build full, qualified name - std::string fullname = mod_name; - fullname += "."; - fullname += cpyrt_PyText_AsString(pyClName); - - Py_XDECREF(pyClName); - } - - Py_DECREF(value); - } - - Py_DECREF(values); - - // TODO: mod "leaks" here - if (PyErr_Occurred()) - return false; - return true; -} - -//----------------------------------------------------------------------------- -void cpyrt::ExecScript(const std::string& name, - const std::vector& args) { - // Execute a python stand-alone script, with argv CLI arguments. - // - // example of use: - // cpyrt::ExecScript("test.py", {"1", "2", "3"}); - - if (!Initialize()) - return; - - PythonGILRAII python_gil_raii; - - // verify arguments - if (name.empty()) { - std::cerr << "Error: no file name specified." << std::endl; - return; - } - - FILE* fp = fopen(name.c_str(), "r"); - if (!fp) { - std::cerr << "Error: could not open file \"" << name << "\"." << std::endl; - return; - } - - // store a copy of the old cli for restoration - PyObject* oldargv = PySys_GetObject("argv"); // borrowed - if (oldargv) { - PyObject* copy = PyList_GetSlice(oldargv, 0, PyList_Size(oldargv)); - oldargv = copy; // now owned - } else { - PyErr_Clear(); - } - - // build new argv - const int argc = (int)args.size() + 1; - std::vector wargv(argc); - wargv[0] = Py_DecodeLocale(name.c_str(), nullptr); - - for (int i = 1; i < argc; ++i) { - wargv[i] = Py_DecodeLocale(args[i - 1].c_str(), nullptr); - } - - // set sys.argv - PyObject* sysmod = PyImport_ImportModule("sys"); // new reference - if (sysmod) { - PyObject* argv_obj = PyList_New(argc); - for (int i = 0; i < argc; ++i) { - PyList_SET_ITEM(argv_obj, i, PyUnicode_FromWideChar(wargv[i], -1)); - } - PyObject_SetAttrString(sysmod, "argv", argv_obj); - Py_DECREF(argv_obj); - Py_DECREF(sysmod); - } else { - PyErr_Print(); - } - - // actual script execution - PyObject* gbl = PyDict_Copy(gMainDict); - PyObject* result = // PyRun_FileEx closes fp (b/c of last argument "1") - PyRun_FileEx(fp, const_cast(name.c_str()), Py_file_input, gbl, gbl, - 1); - - if (!result) - PyErr_Print(); - - Py_XDECREF(result); - Py_DECREF(gbl); - - // restore original command line - if (oldargv) { - PySys_SetObject("argv", oldargv); - Py_DECREF(oldargv); - } - - // free memory from Py_DecodeLocale - for (auto ptr : wargv) - PyMem_RawFree(ptr); -} - //----------------------------------------------------------------------------- -bool cpyrt::Exec(const std::string& cmd) { - // Execute a python statement (e.g. "import noddy"). +PyObject* cpyrt::GetThisModule() { + // Return the cppjit extension module as a borrowed reference. Frameworks + // embedding Python (e.g. TPython) can use it to attach imported python + // modules, so that no python proxy is created for the C++ proxy. if (!Initialize()) - return false; - - PythonGILRAII python_gil_raii; - // execute the command - PyObject* result = PyRun_String(const_cast(cmd.c_str()), Py_file_input, - gMainDict, gMainDict); - - // test for error - if (result) { - Py_DECREF(result); - return true; - } - - PyErr_Print(); - return false; -} - -//----------------------------------------------------------------------------- -void cpyrt::Prompt() { - // Enter an interactive python session (exit with ^D). State is preserved - // between successive calls. - if (!Initialize()) - return; - - PythonGILRAII python_gil_raii; - // enter i/o interactive mode - PyRun_InteractiveLoop(stdin, const_cast("\0")); + return nullptr; + return gThisModule; } diff --git a/bindings/pyroot/cppjit/src/cpyrt/API.h b/bindings/pyroot/cppjit/src/cpyrt/API.h index 627132a2e265b..5bf5b308a7b31 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/API.h +++ b/bindings/pyroot/cppjit/src/cpyrt/API.h @@ -2,7 +2,7 @@ #define CPYRT_API_H // -// Access to the python interpreter and API onto cpyrt. +// API onto cpyrt: access to cppjit proxies from Cling and C++. // // Python @@ -52,7 +52,6 @@ typedef void* TCppFuncAddr_t; // Standard #include -#include namespace cppjit::cpyrt { @@ -232,20 +231,9 @@ CPYRT_EXTERN bool Overload_CheckExact(PyObject* pyobject); // support IO of arbitrary C++ objects, like ROOT. CPYRT_EXTERN void Instance_SetReduceMethod(PyCFunction reduceMethod); -//- access to the python interpreter ---------------------------------------- - -// import a python module, making its classes available to Cling -CPYRT_EXTERN bool Import(const std::string& name); - -// execute a python statement (e.g. "import sys") -CPYRT_EXTERN bool Exec(const std::string& cmd); - -// execute a python stand-alone script, with argv CLI arguments -CPYRT_EXTERN void ExecScript(const std::string& name, - const std::vector& args); - -// enter an interactive python session (exit with ^D) -CPYRT_EXTERN void Prompt(); +// the cppjit extension module itself, as a borrowed reference; frameworks +// embedding Python (e.g. TPython) can attach imported python modules to it +CPYRT_EXTERN PyObject* GetThisModule(); } // namespace cppjit::cpyrt diff --git a/bindings/pyroot/cppjit/src/cpyrt/Utility.cxx b/bindings/pyroot/cppjit/src/cpyrt/Utility.cxx index f83d39c27fcd7..0939fa2a98d4e 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/Utility.cxx +++ b/bindings/pyroot/cppjit/src/cpyrt/Utility.cxx @@ -1600,7 +1600,10 @@ bool cpyrt::Utility::IncludePython() { // utilities from the cpyrt public API "#include \"cpyrt/DispatchPtr.h\"\n" - "#include \"cpyrt/PyException.h\"\n"); + "#include \"cpyrt/PyException.h\"\n" + + // used by the generated callback code + "#include \n"); includesDone = okay; } diff --git a/bindings/pyroot/cppjit/test/test_api.py b/bindings/pyroot/cppjit/test/test_api.py index f6f49262ba9a5..464480746a769 100644 --- a/bindings/pyroot/cppjit/test/test_api.py +++ b/bindings/pyroot/cppjit/test/test_api.py @@ -36,16 +36,6 @@ class APICheck { assert API.Overload_Check(m) assert API.Overload_CheckExact(m) - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") - def test02_interpreter_access(self): - """Access to the python interpreter""" - - import cppjit - - API = cppjit.gbl.cppjit.cpyrt - - assert API.Exec("import sys") - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test03_instance_conversion(self): """Proxy object conversions""" diff --git a/bindings/tpython/CMakeLists.txt b/bindings/tpython/CMakeLists.txt index fafae367ef9f6..2c6196663a98b 100644 --- a/bindings/tpython/CMakeLists.txt +++ b/bindings/tpython/CMakeLists.txt @@ -30,14 +30,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTPython Python3::Python ) -# On Windows we can't use the stable ABI yet: it requires linking against a -# different libpython, so as long as we don't build all translation units in -# the ROOT Pythonization library with the stable ABI we should not use it. -# Free-threaded (Py_GIL_DISABLED) interpreters don't support the limited API. -if(NOT MSVC AND NOT Python3_GIL_DISABLED) - target_compile_options(ROOTTPython PRIVATE -DPy_LIMITED_API=0x030A0000) -endif() - if(MSVC) target_link_options(ROOTTPython PRIVATE "-ignore:4197") endif() diff --git a/bindings/tpython/src/PyGILRAII.h b/bindings/tpython/src/PyGILRAII.h new file mode 100644 index 0000000000000..8602fd700b441 --- /dev/null +++ b/bindings/tpython/src/PyGILRAII.h @@ -0,0 +1,26 @@ +// Author: the ROOT team, CERN + +/************************************************************************* + * Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +#ifndef ROOT_TPython_PyGILRAII +#define ROOT_TPython_PyGILRAII + +#include + +/// Acquire the Python GIL for the current scope. +/// See https://docs.python.org/3/c-api/init.html#non-python-created-threads +class PyGILRAII { + PyGILState_STATE fGILState; + +public: + PyGILRAII() : fGILState(PyGILState_Ensure()) {} + ~PyGILRAII() { PyGILState_Release(fGILState); } +}; + +#endif // ROOT_TPython_PyGILRAII diff --git a/bindings/tpython/src/TPyArg.cxx b/bindings/tpython/src/TPyArg.cxx index efb0a468fe605..7b30c71dfe934 100644 --- a/bindings/tpython/src/TPyArg.cxx +++ b/bindings/tpython/src/TPyArg.cxx @@ -13,6 +13,7 @@ #include #include "TPyArg.h" +#include "PyGILRAII.h" //______________________________________________________________________________ // Generic wrapper for arguments @@ -23,17 +24,6 @@ // In principle, there should be no need to use this class directly: it relies // on implicit conversions. -//- data --------------------------------------------------------------------- - -namespace { - class PyGILRAII { - PyGILState_STATE m_GILState; - public: - PyGILRAII() : m_GILState(PyGILState_Ensure()) { } - ~PyGILRAII() { PyGILState_Release(m_GILState); } - }; -} - //- constructor dispatcher --------------------------------------------------- void TPyArg::CallConstructor(PyObject *&pyself, PyObject *pyclass, const std::vector &args) { diff --git a/bindings/tpython/src/TPyClassGenerator.cxx b/bindings/tpython/src/TPyClassGenerator.cxx index f95163a07e127..ad9fca210f6df 100644 --- a/bindings/tpython/src/TPyClassGenerator.cxx +++ b/bindings/tpython/src/TPyClassGenerator.cxx @@ -13,6 +13,7 @@ #include "TPyClassGenerator.h" #include "TPyReturn.h" +#include "PyGILRAII.h" // ROOT #include "TClass.h" @@ -27,14 +28,6 @@ namespace { -class PyGILRAII { - PyGILState_STATE m_GILState; - -public: - PyGILRAII() : m_GILState(PyGILState_Ensure()) {} - ~PyGILRAII() { PyGILState_Release(m_GILState); } -}; - #if (defined(Py_LIMITED_API) || PY_VERSION_HEX < 0x30d00f0) // Implementation of PyObject_GetOptionalAttr and diff --git a/bindings/tpython/src/TPyReturn.cxx b/bindings/tpython/src/TPyReturn.cxx index 38f926d835373..fd9346a181c6b 100644 --- a/bindings/tpython/src/TPyReturn.cxx +++ b/bindings/tpython/src/TPyReturn.cxx @@ -14,6 +14,7 @@ // Bindings #include "cpyrt/API.h" #include "TPyReturn.h" +#include "PyGILRAII.h" // ROOT #include "TObject.h" @@ -41,17 +42,6 @@ // root [4] d // (double)4.14150000000000063e+00 -//- data --------------------------------------------------------------------- - -namespace { - class PyGILRAII { - PyGILState_STATE m_GILState; - public: - PyGILRAII() : m_GILState(PyGILState_Ensure()) { } - ~PyGILRAII() { PyGILState_Release(m_GILState); } - }; -} - //- constructors/destructor -------------------------------------------------- TPyReturn::TPyReturn() { diff --git a/bindings/tpython/src/TPython.cxx b/bindings/tpython/src/TPython.cxx index fb81f1977ab0d..3e46b1e70bb21 100644 --- a/bindings/tpython/src/TPython.cxx +++ b/bindings/tpython/src/TPython.cxx @@ -12,11 +12,12 @@ #include // Bindings -// cpyrt.h must be go first, since it includes Python.h, which must be -// included before any standard header +// cpyrt/API.h includes Python.h, which must be included before any standard +// header #include "cpyrt/API.h" #include "TPython.h" #include "TPyClassGenerator.h" +#include "PyGILRAII.h" // ROOT #include "TROOT.h" @@ -29,6 +30,7 @@ #include #include #include +#include /// \class TPython /// Accessing the Python interpreter from C++. @@ -99,18 +101,6 @@ static PyObject *gMainDict = 0; namespace { -PyThreadState *mainThreadState; - -// To acquire the GIL as described here: -// https://docs.python.org/3/c-api/init.html#non-python-created-threads -class PyGILRAII { - PyGILState_STATE m_GILState; - -public: - PyGILRAII() : m_GILState(PyGILState_Ensure()) {} - ~PyGILRAII() { PyGILState_Release(m_GILState); } -}; - struct PyObjDeleter { void operator()(PyObject* obj) const { Py_DecRef(obj); @@ -135,10 +125,23 @@ Bool_t TPython::Initialize() return true; if (!Py_IsInitialized()) { - // Trigger the Python initialization indirectly via cpyrt - cppjit::cpyrt::Scope_Check(nullptr); + // This happens if Cling comes in first: embed the Python interpreter + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyConfig_SetString(&config, &config.program_name, L"TPython"); + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + + if (!Py_IsInitialized()) { + // give up ... + std::cerr << "Error: python has not been initialized; returning." << std::endl; + return false; + } - mainThreadState = PyEval_SaveThread(); + // Release the GIL so that other threads can use the interpreter. The + // saved thread state is never restored: ROOT never finalizes the + // embedded interpreter. + (void)PyEval_SaveThread(); } { @@ -202,15 +205,25 @@ Bool_t TPython::Import(const char *mod_name) PyGILRAII gilRaii; - if (!cppjit::cpyrt::Import(mod_name)) { + PyObjectRef mod{PyImport_ImportModule(mod_name)}; + if (!mod) { + PyErr_Print(); return false; } + // attach to the cppjit module to prevent the creation of a python proxy + // for the C++ proxy + if (PyObject *thisModule = cppjit::cpyrt::GetThisModule()) { + Py_IncRef(mod.get()); // PyModule_AddObject steals a reference + PyModule_AddObject(thisModule, mod_name, mod.get()); + } else { + std::cerr << "Warning: cannot attach module \"" << mod_name << "\" to the cppjit module (module not loaded)" + << std::endl; + } + // force creation of the module as a namespace TClass::GetClass(mod_name, true); - PyObjectRef modNameObj{PyUnicode_FromString(mod_name)}; - PyObjectRef mod{PyImport_GetModule(modNameObj.get())}; PyObject *dct = PyModule_GetDict(mod.get()); PyObjectRef basesStr{PyUnicode_FromString("__bases__")}; @@ -347,11 +360,62 @@ void TPython::ExecScript(const char *name, int argc, const char **argv) return; } - std::vector args(argc); - for (int i = 0; i < argc; ++i) { - args[i] = argv[i]; + FILE *fp = fopen(name, "r"); + if (!fp) { + std::cerr << "Error: could not open file \"" << name << "\"." << std::endl; + return; + } + + // store a copy of the old cli for restoration + PyObject *oldargv = PySys_GetObject("argv"); // borrowed + if (oldargv) { + oldargv = PyList_GetSlice(oldargv, 0, PyList_Size(oldargv)); // now owned + } else { + PyErr_Clear(); } - cppjit::cpyrt::ExecScript(name, args); + + // build new argv + const int wargc = argc + 1; + std::vector wargv(wargc); + wargv[0] = Py_DecodeLocale(name, nullptr); + for (int i = 1; i < wargc; ++i) { + wargv[i] = Py_DecodeLocale(argv[i - 1], nullptr); + } + + // set sys.argv + PyObject *sysmod = PyImport_ImportModule("sys"); // new reference + if (sysmod) { + PyObject *argvObj = PyList_New(wargc); + for (int i = 0; i < wargc; ++i) { + PyList_SET_ITEM(argvObj, i, PyUnicode_FromWideChar(wargv[i], -1)); + } + PyObject_SetAttrString(sysmod, "argv", argvObj); + Py_DecRef(argvObj); + Py_DecRef(sysmod); + } else { + PyErr_Print(); + } + + // actual script execution + PyObject *gbl = PyDict_Copy(gMainDict); + PyObject *pyresult = // PyRun_FileEx closes fp (b/c of last argument "1") + PyRun_FileEx(fp, const_cast(name), Py_file_input, gbl, gbl, 1); + + if (!pyresult) + PyErr_Print(); + else + Py_DecRef(pyresult); + Py_DecRef(gbl); + + // restore original command line + if (oldargv) { + PySys_SetObject("argv", oldargv); + Py_DecRef(oldargv); + } + + // free memory from Py_DecodeLocale + for (auto ptr : wargv) + PyMem_RawFree(ptr); } //////////////////////////////////////////////////////////////////////////////// @@ -394,7 +458,14 @@ Bool_t TPython::Exec(const char *cmd, std::any *result, std::string const &resul } // execute the command - return cppjit::cpyrt::Exec(command.str()); + PyObject *pyresult = PyRun_String(command.str().c_str(), Py_file_input, gMainDict, gMainDict); + if (pyresult) { + Py_DecRef(pyresult); + return true; + } + + PyErr_Print(); + return false; } //////////////////////////////////////////////////////////////////////////////// @@ -437,7 +508,7 @@ void TPython::Prompt() PyGILRAII gilRaii; // enter i/o interactive mode - cppjit::cpyrt::Prompt(); + PyRun_InteractiveLoop(stdin, const_cast("\0")); } ////////////////////////////////////////////////////////////////////////////////