gh-132312: Significantly reduce thread state lookups in the interpreter

Include/cpython/object.h

@@ -317,9 +317,9 @@ PyAPI_FUNC(void) PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *);
 /* Same as PyObject_Generic{Get,Set}Attr, but passing the attributes
    dict as the last parameter. */
 PyAPI_FUNC(PyObject *)
-_PyObject_GenericGetAttrWithDict(PyObject *, PyObject *, PyObject *, int);
+_PyObject_GenericGetAttrWithDict(PyThreadState *, PyObject *, PyObject *, PyObject *, int);
 PyAPI_FUNC(int)
-_PyObject_GenericSetAttrWithDict(PyObject *, PyObject *,
+_PyObject_GenericSetAttrWithDict(PyThreadState *, PyObject *, PyObject *,
                                  PyObject *, PyObject *);
 
 PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *);
@@ -388,6 +388,45 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *);
     } while (0)
 #endif
 
+#ifdef _Py_TYPEOF
+#define _Py_SETREF(tstate, dst, src) \
+    do { \
+        _Py_TYPEOF(dst)* _tmp_dst_ptr = &(dst); \
+        _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
+        *_tmp_dst_ptr = (src); \
+        _Py_DECREF(tstate, _tmp_old_dst); \
+    } while (0)
+#else
+#define _Py_SETREF(tstate, dst, src) \
+    do { \
+        PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
+        PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
+        PyObject *_tmp_src = _PyObject_CAST(src); \
+        memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
+        _Py_DECREF(tstate, _tmp_old_dst); \
+    } while (0)
+#endif
+
+
+#ifdef _Py_TYPEOF
+#define _Py_XSETREF(tstate, dst, src) \
+    do { \
+        _Py_TYPEOF(dst)* _tmp_dst_ptr = &(dst); \
+        _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
+        *_tmp_dst_ptr = (src); \
+        _Py_XDECREF(tstate, _tmp_old_dst); \
+    } while (0)
+#else
+#define _Py_XSETREF(tstate, dst, src) \
+    do { \
+        PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
+        PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
+        PyObject *_tmp_src = _PyObject_CAST(src); \
+        memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
+        _Py_XDECREF(tstate, _tmp_old_dst); \
+    } while (0)
+#endif
+
 
 /* Define a pair of assertion macros:
    _PyObject_ASSERT_FROM(), _PyObject_ASSERT_WITH_MSG() and _PyObject_ASSERT().

Include/internal/pycore_call.h

@@ -46,7 +46,7 @@ extern PyObject* _PyObject_VectorcallDictTstate(
     size_t nargsf,
     PyObject *kwargs);
 
-extern PyObject* _PyObject_Call(
+PyAPI_FUNC(PyObject*) _PyObject_Call(
     PyThreadState *tstate,
     PyObject *callable,
     PyObject *args,
@@ -176,6 +176,9 @@ PyAPI_FUNC(void) _PyStack_UnpackDict_FreeNoDecRef(
     PyObject *const *stack,
     PyObject *kwnames);
 
+PyAPI_FUNC(PyObject *)
+_PyObject_CallOneArgTstate(PyThreadState *tstate, PyObject *func, PyObject *arg);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_ceval.h

@@ -328,7 +328,7 @@ PyAPI_FUNC(PyObject *) _PyEval_GetAwaitable(PyObject *iterable, int oparg);
 PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name);
 PyAPI_FUNC(int)
 _Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args);
-PyAPI_FUNC(_PyStackRef) _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *null_or_index, int yield_from);
+PyAPI_FUNC(_PyStackRef) _PyEval_GetIter(PyThreadState *tstate, _PyStackRef iterable, _PyStackRef *null_or_index, int yield_from);
 
 /*
  * Indicate whether a special method of given 'oparg' can use the (improved)
@@ -411,6 +411,7 @@ PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR;
 
 PyAPI_FUNC(PyObject *)
 _Py_VectorCall_StackRefSteal(
+    PyThreadState *tstate,
     _PyStackRef callable,
     _PyStackRef *arguments,
     int total_args,

Include/internal/pycore_moduleobject.h

@@ -77,7 +77,7 @@ extern Py_ssize_t _PyModule_GetFilenameUTF8(
         char *buffer,
         Py_ssize_t maxlen);
 
-PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress);
+PyObject* _Py_module_getattro_impl(PyThreadState *tstate, PyModuleObject *m, PyObject *name, int suppress);
 PyObject* _Py_module_getattro(PyObject *m, PyObject *name);
 
 #ifdef __cplusplus

Include/internal/pycore_object.h

@@ -420,6 +420,7 @@ static inline void Py_DECREF_MORTAL(const char *filename, int lineno, PyObject *
     }
 }
 #define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(__FILE__, __LINE__, _PyObject_CAST(op))
+#define _Py_DECREF_MORTAL(tstate, op) Py_DECREF_MORTAL(op)
 
 static inline void _Py_DECREF_MORTAL_SPECIALIZED(const char *filename, int lineno, PyObject *op, destructor destruct)
 {
@@ -443,6 +444,16 @@ static inline void _Py_DECREF_MORTAL_SPECIALIZED(const char *filename, int linen
 
 #else
 
+static inline void _Py_DECREF_MORTAL(PyThreadState *tstate, PyObject *op)
+{
+    assert(!_Py_IsStaticImmortal(op));
+    _Py_DECREF_STAT_INC();
+    if (--op->ob_refcnt == 0) {
+        _Py_DeallocTstate(tstate, op);
+    }
+}
+#define _Py_DECREF_MORTAL(tstate, op) _Py_DECREF_MORTAL(tstate, _PyObject_CAST(op))
+
 static inline void Py_DECREF_MORTAL(PyObject *op)
 {
     assert(!_Py_IsStaticImmortal(op));
@@ -466,6 +477,7 @@ static inline void Py_DECREF_MORTAL_SPECIALIZED(PyObject *op, destructor destruc
 
 #endif
 #else  // Py_GIL_DISABLED
+# define _Py_DECREF_MORTAL(tstate, op) _Py_DECREF(tstate, op)
 # define Py_DECREF_MORTAL(op) Py_DECREF(op)
 # define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF(op)
 #endif
@@ -854,7 +866,7 @@ _PyType_PreHeaderSize(PyTypeObject *tp)
     );
 }
 
-void _PyObject_GC_Link(PyObject *op);
+void _PyObject_GC_Link(PyThreadState *tstate, PyObject *op);
 
 // Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL));
 extern int _Py_CheckSlotResult(
@@ -874,7 +886,7 @@ extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
 extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *);
 extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int);
 // Exported for external JIT support
-PyAPI_FUNC(int) _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name);
+PyAPI_FUNC(int) _PyObject_SetAttributeErrorContext(PyThreadState *tstate, PyObject *v, PyObject* name);
 
 void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
 extern int _PyObject_StoreInstanceAttribute(PyObject *obj,
@@ -896,7 +908,7 @@ extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self,
 
 // Like PyObject_GetAttr but returns a _PyStackRef. For types, this can
 // return a deferred reference to reduce reference count contention.
-PyAPI_FUNC(_PyStackRef) _PyObject_GetAttrStackRef(PyObject *obj, PyObject *name);
+PyAPI_FUNC(_PyStackRef) _PyObject_GetAttrStackRef(PyThreadState *tstate, PyObject *obj, PyObject *name);
 
 // Cache the provided init method in the specialization cache of type if the
 // provided type version matches the current version of the type.
@@ -965,7 +977,7 @@ _PyObject_MaybeCallSpecialOneArg(PyObject *self, PyObject *attr, PyObject *arg);
 
 extern int _PyObject_IsAbstract(PyObject *);
 
-PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method);
+PyAPI_FUNC(int) _PyObject_GetMethod(PyThreadState *tstate, PyObject *obj, PyObject *name, PyObject **method);
 extern PyObject* _PyObject_NextNotImplemented(PyObject *);
 
 // Pickle support.
@@ -1033,6 +1045,10 @@ static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op)
  * references. */
 PyAPI_FUNC(int) _PyObject_VisitType(PyObject *op, visitproc visit, void *arg);
 
+PyAPI_FUNC(PyObject *) _PyObject_RichCompare(PyThreadState *, PyObject *, PyObject *, int);
+PyAPI_FUNC(int) _PyObject_RichCompareBool(PyThreadState *, PyObject *, PyObject *, int);
+PyAPI_FUNC(PyObject *) _PyObject_GetAttr(PyThreadState *, PyObject *, PyObject *);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_pyerrors.h

@@ -211,6 +211,12 @@ extern int _PyUnicodeError_GetParams(
     Py_ssize_t *slen,
     int as_bytes);
 
+PyAPI_FUNC(void)
+_PyErr_FormatUnraisable(PyThreadState *tstate, const char *format, ...);
+
+PyAPI_FUNC(void)
+_PyErr_WriteUnraisable(PyThreadState *tstate, PyObject *obj);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_stackref.h

@@ -213,7 +213,7 @@ _PyStackRef_FromPyObjectBorrow(PyObject *obj, const char *filename, int linenumb
 #define PyStackRef_FromPyObjectBorrow(obj) _PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj), __FILE__, __LINE__)
 
 static inline void
-_PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
+_PyStackRef_CloseWithFile(_PyStackRef ref, const char *filename, int linenumber)
 {
     assert(!PyStackRef_IsError(ref));
     assert(!PyStackRef_IsNull(ref));
@@ -226,18 +226,18 @@ _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
         Py_DECREF(obj);
     }
 }
-#define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__)
+#define PyStackRef_CLOSE(REF) _PyStackRef_CloseWithFile((REF), __FILE__, __LINE__)
 
 static inline void
-_PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)
+_PyStackRef_XCloseWithFile(_PyStackRef ref, const char *filename, int linenumber)
 {
     assert(!PyStackRef_IsError(ref));
     if (PyStackRef_IsNull(ref)) {
         return;
     }
     _PyStackRef_CLOSE(ref, filename, linenumber);
 }
-#define PyStackRef_XCLOSE(REF) _PyStackRef_XCLOSE((REF), __FILE__, __LINE__)
+#define PyStackRef_XCLOSE(REF) _PyStackRef_XCloseWithFile((REF), __FILE__, __LINE__)
 
 static inline _PyStackRef
 _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
@@ -694,7 +694,21 @@ do { \
     _PyStackRef _temp = (REF); \
     if (PyStackRef_RefcountOnObject(_temp)) Py_DECREF_MORTAL(BITS_TO_PTR(_temp)); \
 } while (0)
+#define _PyStackRef_CLOSE(tstate, REF) \
+do { \
+    _PyStackRef _temp = (REF); \
+    if (PyStackRef_RefcountOnObject(_temp)) _Py_DECREF_MORTAL(tstate, BITS_TO_PTR(_temp)); \
+} while (0)
 #else
+static inline void
+_PyStackRef_CLOSE(PyThreadState *tstate, _PyStackRef ref)
+{
+    assert(!PyStackRef_IsNull(ref));
+    if (PyStackRef_RefcountOnObject(ref)) {
+        _Py_DECREF_MORTAL(tstate, BITS_TO_PTR(ref));
+    }
+}
+
 static inline void
 PyStackRef_CLOSE(_PyStackRef ref)
 {
@@ -716,7 +730,18 @@ PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
 
 #ifdef _WIN32
 #define PyStackRef_XCLOSE PyStackRef_CLOSE
+#define _PyStackRef_XCLOSE _PyStackRef_CLOSE
 #else
+static inline void
+_PyStackRef_XCLOSE(PyThreadState *tstate, _PyStackRef ref)
+{
+    assert(ref.bits != 0);
+    if (PyStackRef_RefcountOnObject(ref)) {
+        assert(!PyStackRef_IsNull(ref));
+        _Py_DECREF_MORTAL(tstate, BITS_TO_PTR(ref));
+    }
+}
+
 static inline void
 PyStackRef_XCLOSE(_PyStackRef ref)
 {
@@ -736,6 +761,13 @@ PyStackRef_XCLOSE(_PyStackRef ref)
         PyStackRef_XCLOSE(_tmp_old_op); \
     } while (0)
 
+#define _PyStackRef_CLEAR(tstate, REF) \
+    do { \
+        _PyStackRef *_tmp_op_ptr = &(REF); \
+        _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
+        *_tmp_op_ptr = PyStackRef_NULL; \
+        _PyStackRef_XCLOSE(tstate, _tmp_old_op); \
+    } while (0)
 
 // Note: this is a macro because MSVC (Windows) has trouble inlining it.
 
@@ -808,7 +840,7 @@ _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref)
     assert(tstate_impl->c_stack_refs == ref);
     tstate_impl->c_stack_refs = ref->next;
 #endif
-    PyStackRef_XCLOSE(ref->ref);
+    _PyStackRef_XCLOSE(tstate, ref->ref);
 }
 
 static inline _PyStackRef
@@ -858,6 +890,13 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out)
         PyStackRef_XCLOSE(_tmp_dst_ref); \
     } while(0)
 
+#define _PyStackRef_XSETREF(tstate, dst, src) \
+    do { \
+        _PyStackRef _tmp_dst_ref = (dst); \
+        (dst) = (src); \
+        _PyStackRef_XCLOSE(tstate, _tmp_dst_ref); \
+    } while(0)
+
 // Like Py_VISIT but for _PyStackRef fields
 #define _Py_VISIT_STACKREF(ref)                                         \
     do {                                                                \

Include/internal/pycore_typeobject.h

@@ -111,10 +111,10 @@ _PyType_IsReady(PyTypeObject *type)
     return _PyType_GetDict(type) != NULL;
 }
 
-extern PyObject* _Py_type_getattro_impl(PyTypeObject *type, PyObject *name,
+extern PyObject* _Py_type_getattro_impl(PyThreadState *tstate, PyTypeObject *type, PyObject *name,
                                         int *suppress_missing_attribute);
 extern PyObject* _Py_type_getattro(PyObject *type, PyObject *name);
-extern _PyStackRef _Py_type_getattro_stackref(PyTypeObject *type, PyObject *name,
+extern _PyStackRef _Py_type_getattro_stackref(PyThreadState *tstate, PyTypeObject *type, PyObject *name,
                                               int *suppress_missing_attribute);
 
 extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int op);

Include/methodobject.h

@@ -117,7 +117,7 @@ PyAPI_FUNC(PyObject *) PyCMethod_New(PyMethodDef *, PyObject *,
 
 /* This bit is preserved for Stackless Python */
 #ifdef STACKLESS
-#  define METH_STACKLESS 0x0100
+#  define METH_STACKLESS 0x0000
 #else
 #  define METH_STACKLESS 0x0000
 #endif