Skip to content

[tpython] Handle Python interpreter embedding in TPython, not cppjit #23616

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
guitargeek wants to merge 3 commits into
base: master
Choose a base branch
from
Open

[tpython] Handle Python interpreter embedding in TPython, not cppjit #23616

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a37e6ce
[tpython] Handle Python interpreter embedding in TPython, not cppjit
guitargeek Oct 5, 2026
69e6be5
[tpython] Clean up GIL handling and diagnostics
guitargeek Oct 5, 2026
9159ea6
[tpython] ROOTTPython cannot use the Python stable ABI
guitargeek Oct 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 13 additions & 202 deletions bindings/pyroot/cppjit/src/cpyrt/API.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,9 @@ using namespace cppjit;
#include "CPPOverload.h"
#include "CPPScope.h"
#include "ProxyWrappers.h"
#include "PyStrings.h"
#include "cpyrt/DispatchPtr.h"

// Standard
#include <iostream>
#include <stdio.h>
#include <string>

//______________________________________________________________________________
Expand All @@ -25,11 +22,6 @@ using namespace cppjit;
//
// Access to cppjit Python objects from Cling and C++: allows conversion for
// instances and type checking for scopes, instances, etc.
// Adds a few convenience functions to call Python from Cling and expose Python
// classes to Cling for use in inheritance etc.

//- data ---------------------------------------------------------------------
static PyObject* gMainDict = nullptr;

namespace cppjit::cpyrt {
extern PyObject* gThisModule;
Expand All @@ -39,27 +31,17 @@ extern PyObject* gThisModule;
namespace {

static bool Initialize() {
// Private initialization method: setup the python interpreter and load the
// cppjit module.
// Private initialization method: load the cppjit module. Expects the Python
// interpreter to be already initialized.
static bool isInitialized = false;
if (isInitialized)
return true;

if (!Py_IsInitialized()) {
// this happens if Cling comes in first
PyConfig config;
PyConfig_InitPythonConfig(&config);
PyConfig_SetString(&config, &config.program_name, L"cppjit");
Py_InitializeFromConfig(&config);

// try again to see if the interpreter is initialized
if (!Py_IsInitialized()) {
// give up ...
std::cerr << "Error: python has not been initialized; returning."
<< std::endl;
return false;
}
}
// cppjit is a Python extension library, not an embedding library: the
// Python interpreter must be initialized by the embedding application
// (e.g. TPython) before any cpyrt API function is called.
if (!Py_IsInitialized())
return false;

// Importing the extension module is what runs the cpyrt initialization
// that sets gThisModule.
Expand All @@ -70,17 +52,6 @@ static bool Initialize() {
Py_DECREF(cppjitmod);
}

if (!gMainDict) {
cpyrt::PythonGILRAII python_gil_raii;
// retrieve the main dictionary
gMainDict =
PyModule_GetDict(PyImport_AddModule(const_cast<char*>("__main__")));
// The gMainDict is borrowed, i.e. we are not calling Py_INCREF(gMainDict).
// Like this, we avoid unexpectedly affecting how long __main__ is kept
// alive. The gMainDict is only used in Exec(), ExecScript(), and Eval(),
// which should not be called after __main__ is garbage collected anyway.
}

// declare success ...
isInitialized = true;
return true;
Expand Down Expand Up @@ -306,172 +277,12 @@ void cpyrt::Instance_SetReduceMethod(PyCFunction reduceMethod) {
CPPInstance::ReduceMethod() = reduceMethod;
}

//- access to the python interpreter ----------------------------------------
bool cpyrt::Import(const std::string& mod_name) {
// Import the named python module and create Cling equivalents for its
// classes.
if (!Initialize())
return false;

PythonGILRAII python_gil_raii;

PyObject* mod = PyImport_ImportModule(mod_name.c_str());
if (!mod) {
PyErr_Print();
return false;
}

// allow finding to prevent creation of a python proxy for the C++ proxy
Py_INCREF(mod);
PyModule_AddObject(gThisModule, mod_name.c_str(), mod);

// force creation of the module as a namespace
// TODO: the following is broken (and should live in cppjit_interop.cxx)
// TClass::GetClass(mod_name, true);

PyObject* dct = PyModule_GetDict(mod);

// create Cling classes for all new python classes
PyObject* values = PyDict_Values(dct);
for (int i = 0; i < PyList_GET_SIZE(values); ++i) {
PyObject* value = PyList_GET_ITEM(values, i);
Py_INCREF(value);

// collect classes
if (PyClass_Check(value) || PyObject_HasAttr(value, PyStrings::gBases)) {
// get full class name (including module)
PyObject* pyClName = PyObject_GetAttr(value, PyStrings::gName);
if (PyErr_Occurred())
PyErr_Clear();

// build full, qualified name
std::string fullname = mod_name;
fullname += ".";
fullname += cpyrt_PyText_AsString(pyClName);

Py_XDECREF(pyClName);
}

Py_DECREF(value);
}

Py_DECREF(values);

// TODO: mod "leaks" here
if (PyErr_Occurred())
return false;
return true;
}

//-----------------------------------------------------------------------------
void cpyrt::ExecScript(const std::string& name,
const std::vector<std::string>& args) {
// Execute a python stand-alone script, with argv CLI arguments.
//
// example of use:
// cpyrt::ExecScript("test.py", {"1", "2", "3"});

if (!Initialize())
return;

PythonGILRAII python_gil_raii;

// verify arguments
if (name.empty()) {
std::cerr << "Error: no file name specified." << std::endl;
return;
}

FILE* fp = fopen(name.c_str(), "r");
if (!fp) {
std::cerr << "Error: could not open file \"" << name << "\"." << std::endl;
return;
}

// store a copy of the old cli for restoration
PyObject* oldargv = PySys_GetObject("argv"); // borrowed
if (oldargv) {
PyObject* copy = PyList_GetSlice(oldargv, 0, PyList_Size(oldargv));
oldargv = copy; // now owned
} else {
PyErr_Clear();
}

// build new argv
const int argc = (int)args.size() + 1;
std::vector<wchar_t*> wargv(argc);
wargv[0] = Py_DecodeLocale(name.c_str(), nullptr);

for (int i = 1; i < argc; ++i) {
wargv[i] = Py_DecodeLocale(args[i - 1].c_str(), nullptr);
}

// set sys.argv
PyObject* sysmod = PyImport_ImportModule("sys"); // new reference
if (sysmod) {
PyObject* argv_obj = PyList_New(argc);
for (int i = 0; i < argc; ++i) {
PyList_SET_ITEM(argv_obj, i, PyUnicode_FromWideChar(wargv[i], -1));
}
PyObject_SetAttrString(sysmod, "argv", argv_obj);
Py_DECREF(argv_obj);
Py_DECREF(sysmod);
} else {
PyErr_Print();
}

// actual script execution
PyObject* gbl = PyDict_Copy(gMainDict);
PyObject* result = // PyRun_FileEx closes fp (b/c of last argument "1")
PyRun_FileEx(fp, const_cast<char*>(name.c_str()), Py_file_input, gbl, gbl,
1);

if (!result)
PyErr_Print();

Py_XDECREF(result);
Py_DECREF(gbl);

// restore original command line
if (oldargv) {
PySys_SetObject("argv", oldargv);
Py_DECREF(oldargv);
}

// free memory from Py_DecodeLocale
for (auto ptr : wargv)
PyMem_RawFree(ptr);
}

//-----------------------------------------------------------------------------
bool cpyrt::Exec(const std::string& cmd) {
// Execute a python statement (e.g. "import noddy").
PyObject* cpyrt::GetThisModule() {
// Return the cppjit extension module as a borrowed reference. Frameworks
// embedding Python (e.g. TPython) can use it to attach imported python
// modules, so that no python proxy is created for the C++ proxy.
if (!Initialize())
return false;

PythonGILRAII python_gil_raii;
// execute the command
PyObject* result = PyRun_String(const_cast<char*>(cmd.c_str()), Py_file_input,
gMainDict, gMainDict);

// test for error
if (result) {
Py_DECREF(result);
return true;
}

PyErr_Print();
return false;
}

//-----------------------------------------------------------------------------
void cpyrt::Prompt() {
// Enter an interactive python session (exit with ^D). State is preserved
// between successive calls.
if (!Initialize())
return;

PythonGILRAII python_gil_raii;
// enter i/o interactive mode
PyRun_InteractiveLoop(stdin, const_cast<char*>("\0"));
return nullptr;
return gThisModule;
}
20 changes: 4 additions & 16 deletions bindings/pyroot/cppjit/src/cpyrt/API.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
#define CPYRT_API_H

//
// Access to the python interpreter and API onto cpyrt.
// API onto cpyrt: access to cppjit proxies from Cling and C++.

@vepadulano vepadulano Oct 6, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does cppjit want to have comments mentioning explicitly Cling?

@guitargeek guitargeek Oct 6, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It did before, the Cling name drop was just moved from API.cxx. But you're right, this is the moment to remove that mention.

//

// Python
Expand Down Expand Up @@ -52,7 +52,6 @@ typedef void* TCppFuncAddr_t;

// Standard
#include <string>
#include <vector>

namespace cppjit::cpyrt {

Expand Down Expand Up @@ -232,20 +231,9 @@ CPYRT_EXTERN bool Overload_CheckExact(PyObject* pyobject);
// support IO of arbitrary C++ objects, like ROOT.
CPYRT_EXTERN void Instance_SetReduceMethod(PyCFunction reduceMethod);

//- access to the python interpreter ----------------------------------------

// import a python module, making its classes available to Cling
CPYRT_EXTERN bool Import(const std::string& name);

// execute a python statement (e.g. "import sys")
CPYRT_EXTERN bool Exec(const std::string& cmd);

// execute a python stand-alone script, with argv CLI arguments
CPYRT_EXTERN void ExecScript(const std::string& name,
const std::vector<std::string>& args);

// enter an interactive python session (exit with ^D)
CPYRT_EXTERN void Prompt();
// the cppjit extension module itself, as a borrowed reference; frameworks
// embedding Python (e.g. TPython) can attach imported python modules to it
CPYRT_EXTERN PyObject* GetThisModule();

} // namespace cppjit::cpyrt

Expand Down
5 changes: 4 additions & 1 deletion bindings/pyroot/cppjit/src/cpyrt/Utility.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1600,7 +1600,10 @@ bool cpyrt::Utility::IncludePython() {

// utilities from the cpyrt public API
"#include \"cpyrt/DispatchPtr.h\"\n"
"#include \"cpyrt/PyException.h\"\n");
"#include \"cpyrt/PyException.h\"\n"

// used by the generated callback code
"#include <vector>\n");
includesDone = okay;
}

Expand Down
10 changes: 0 additions & 10 deletions bindings/pyroot/cppjit/test/test_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,6 @@ class APICheck {
assert API.Overload_Check(m)
assert API.Overload_CheckExact(m)

@mark.xfail(condition=IS_MAC, reason="Fails on OS X")
def test02_interpreter_access(self):
"""Access to the python interpreter"""

import cppjit

API = cppjit.gbl.cppjit.cpyrt

assert API.Exec("import sys")

@mark.xfail(condition=IS_MAC, reason="Fails on OS X")
def test03_instance_conversion(self):
"""Proxy object conversions"""
Expand Down
8 changes: 0 additions & 8 deletions bindings/tpython/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTPython
Python3::Python
)

# On Windows we can't use the stable ABI yet: it requires linking against a
# different libpython, so as long as we don't build all translation units in
# the ROOT Pythonization library with the stable ABI we should not use it.
# Free-threaded (Py_GIL_DISABLED) interpreters don't support the limited API.
if(NOT MSVC AND NOT Python3_GIL_DISABLED)
target_compile_options(ROOTTPython PRIVATE -DPy_LIMITED_API=0x030A0000)
endif()

if(MSVC)
target_link_options(ROOTTPython PRIVATE "-ignore:4197")
endif()
Expand Down
26 changes: 26 additions & 0 deletions bindings/tpython/src/PyGILRAII.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Author: the ROOT team, CERN
Comment thread
vepadulano marked this conversation as resolved.

/*************************************************************************
* Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. *
* All rights reserved. *
* *
* For the licensing terms see $ROOTSYS/LICENSE. *
* For the list of contributors see $ROOTSYS/README/CREDITS. *
*************************************************************************/
Comment thread
vepadulano marked this conversation as resolved.

#ifndef ROOT_TPython_PyGILRAII
#define ROOT_TPython_PyGILRAII

#include <Python.h>

/// Acquire the Python GIL for the current scope.
/// See https://docs.python.org/3/c-api/init.html#non-python-created-threads
class PyGILRAII {
PyGILState_STATE fGILState;

public:
PyGILRAII() : fGILState(PyGILState_Ensure()) {}
~PyGILRAII() { PyGILState_Release(fGILState); }
};

#endif // ROOT_TPython_PyGILRAII
12 changes: 1 addition & 11 deletions bindings/tpython/src/TPyArg.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <Python.h>

#include "TPyArg.h"
#include "PyGILRAII.h"

//______________________________________________________________________________
// Generic wrapper for arguments
Expand All @@ -23,17 +24,6 @@
// In principle, there should be no need to use this class directly: it relies
// on implicit conversions.

//- data ---------------------------------------------------------------------

namespace {
class PyGILRAII {
PyGILState_STATE m_GILState;
public:
PyGILRAII() : m_GILState(PyGILState_Ensure()) { }
~PyGILRAII() { PyGILState_Release(m_GILState); }
};
}

//- constructor dispatcher ---------------------------------------------------
void TPyArg::CallConstructor(PyObject *&pyself, PyObject *pyclass, const std::vector<TPyArg> &args)
{
Expand Down
Loading
Loading