gh-149965: promote ptr_wise_atomic_memmove to internal header

Include/internal/pycore_object.h

@@ -1039,6 +1039,35 @@ static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op)
  * references. */
 PyAPI_FUNC(int) _PyObject_VisitType(PyObject *op, visitproc visit, void *arg);
 
+// Pointer-by-pointer memmove for PyObject** arrays that is safe for shared
+// objects in Py_GIL_DISABLED builds.  Locking is the caller's responsibility.
+static inline void
+_Py_ptr_wise_atomic_memmove(PyObject *a, PyObject **dest, PyObject **src,
+                             Py_ssize_t n)
+{
+#ifndef Py_GIL_DISABLED
+    (void)a;
+    memmove(dest, src, n * sizeof(PyObject *));
+#else
+    if (_Py_IsOwnedByCurrentThread(a) && !_PyObject_GC_IS_SHARED(a)) {
+        // No other threads can read this object's array concurrently
+        memmove(dest, src, n * sizeof(PyObject *));
+        return;
+    }
+    if (dest < src) {
+        for (Py_ssize_t i = 0; i != n; i++) {
+            _Py_atomic_store_ptr_release(&dest[i], src[i]);
+        }
+    }
+    else {
+        // copy backwards to avoid overwriting src before it's read
+        for (Py_ssize_t i = n; i != 0; i--) {
+            _Py_atomic_store_ptr_release(&dest[i - 1], src[i - 1]);
+        }
+    }
+#endif
+}
+
 #ifdef __cplusplus
 }
 #endif

Modules/Setup.stdlib.in

@@ -172,7 +172,7 @@
 @MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
-@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c
+@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/test_ptr_wise_memmove.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c
 @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c
 @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c

Modules/_elementtree.c

@@ -18,6 +18,7 @@
 #include "Python.h"
 #include "pycore_ceval.h"         // _Py_EnterRecursiveCall()
 #include "pycore_dict.h"          // _PyDict_CopyAsDict()
+#include "pycore_object.h"        // _Py_ptr_wise_atomic_memmove()
 #include "pycore_pyhash.h"        // _Py_HashSecret
 #include "pycore_tuple.h"         // _PyTuple_FromPair
 #include "pycore_weakref.h"       // FT_CLEAR_WEAKREFS()
@@ -1863,6 +1864,7 @@ element_subscr(PyObject *op, PyObject *item)
     }
 }
 
+
 static int
 element_ass_subscr(PyObject *op, PyObject *item, PyObject *value)
 {
@@ -1938,19 +1940,21 @@ element_ass_subscr(PyObject *op, PyObject *item, PyObject *value)
 
                 PyList_SET_ITEM(recycle, i, self->extra->children[cur]);
 
-                memmove(
+                _Py_ptr_wise_atomic_memmove(
+                    (PyObject *)self,
                     self->extra->children + cur - i,
                     self->extra->children + cur + 1,
-                    num_moved * sizeof(PyObject *));
+                    num_moved);
             }
 
             /* Leftover "tail" after the last removed child */
             cur = start + (size_t)slicelen * step;
             if (cur < (size_t)self->extra->length) {
-                memmove(
+                _Py_ptr_wise_atomic_memmove(
+                    (PyObject *)self,
                     self->extra->children + cur - slicelen,
                     self->extra->children + cur,
-                    (self->extra->length - cur) * sizeof(PyObject *));
+                    self->extra->length - cur);
             }
 
             self->extra->length -= slicelen;

Modules/_testinternalcapi.c

@@ -3340,6 +3340,9 @@ module_exec(PyObject *module)
     if (_PyTestInternalCapi_Init_PyTime(module) < 0) {
         return 1;
     }
+    if (_PyTestInternalCapi_Init_PtrWiseMemmove(module) < 0) {
+        return 1;
+    }
     if (_PyTestInternalCapi_Init_Set(module) < 0) {
         return 1;
     }

Modules/_testinternalcapi/parts.h

@@ -12,6 +12,7 @@
 
 int _PyTestInternalCapi_Init_Lock(PyObject *module);
 int _PyTestInternalCapi_Init_PyTime(PyObject *module);
+int _PyTestInternalCapi_Init_PtrWiseMemmove(PyObject *module);
 int _PyTestInternalCapi_Init_Set(PyObject *module);
 int _PyTestInternalCapi_Init_Complex(PyObject *module);
 int _PyTestInternalCapi_Init_CriticalSection(PyObject *module);

Modules/_testinternalcapi/test_ptr_wise_memmove.c

@@ -0,0 +1,212 @@
+// Tests for _Py_ptr_wise_atomic_memmove() in pycore_object.h
+
+#include "parts.h"
+#include "pycore_object.h"    // _Py_ptr_wise_atomic_memmove()
+#include "pycore_gc.h"        // _PyObject_GC_SET_SHARED()
+
+// Five distinguishable immortal singletons used as placeholder pointers.
+// These require no reference-count management when stored in a raw array.
+#define NPTRS 5
+static PyObject *test_objs[NPTRS];
+
+static void
+setup_test_objs(void)
+{
+    test_objs[0] = Py_None;
+    test_objs[1] = Py_True;
+    test_objs[2] = Py_False;
+    test_objs[3] = Py_Ellipsis;
+    test_objs[4] = Py_NotImplemented;
+}
+
+// Fill buf[0..NPTRS-1] with test_objs in order.
+static void
+fill_buf(PyObject **buf)
+{
+    for (int i = 0; i < NPTRS; i++) {
+        buf[i] = test_objs[i];
+    }
+}
+
+// Return a fresh empty PyListObject whose GC SHARED bit is set in
+// Py_GIL_DISABLED builds.  This forces _Py_ptr_wise_atomic_memmove()
+// to take the atomic (non-fast) path so we can exercise all branches.
+// In non-GIL builds the function always uses memmove, so no flag is needed.
+static PyObject *
+make_shared_container(void)
+{
+    PyObject *a = PyList_New(0);
+    if (a == NULL) {
+        return NULL;
+    }
+#ifdef Py_GIL_DISABLED
+    _PyObject_GC_SET_SHARED(a);
+#endif
+    return a;
+}
+
+// Helper: create container, call the function, return it for cleanup.
+// Returns NULL (with exception set) on allocation failure.
+static PyObject *
+call_memmove(PyObject **dest, PyObject **src, Py_ssize_t n)
+{
+    PyObject *a = make_shared_container();
+    if (a != NULL) {
+        _Py_ptr_wise_atomic_memmove(a, dest, src, n);
+    }
+    return a;
+}
+
+
+// dest < src: forward pointer-by-pointer copy.
+// buf = [0,1,2,3,4], copy src=&buf[2] n=3 to dest=&buf[0]
+// Expected result: buf = [2,3,4,3,4]
+static PyObject *
+test_memmove_dest_lt_src(PyObject *self, PyObject *Py_UNUSED(arg))
+{
+    setup_test_objs();
+
+    PyObject *buf[NPTRS];
+    fill_buf(buf);
+
+    PyObject *a = call_memmove(&buf[0], &buf[2], 3);
+    if (a == NULL) {
+        return NULL;
+    }
+
+    assert(buf[0] == test_objs[2]);
+    assert(buf[1] == test_objs[3]);
+    assert(buf[2] == test_objs[4]);
+    assert(buf[3] == test_objs[3]);  // unchanged
+    assert(buf[4] == test_objs[4]);  // unchanged
+
+    Py_DECREF(a);
+    Py_RETURN_NONE;
+}
+
+// dest > src: backward pointer-by-pointer copy.
+// buf = [0,1,2,3,4], copy src=&buf[0] n=3 to dest=&buf[2]
+// Expected result: buf = [0,1,0,1,2]
+static PyObject *
+test_memmove_dest_gt_src(PyObject *self, PyObject *Py_UNUSED(arg))
+{
+    setup_test_objs();
+
+    PyObject *buf[NPTRS];
+    fill_buf(buf);
+
+    PyObject *a = call_memmove(&buf[2], &buf[0], 3);
+    if (a == NULL) {
+        return NULL;
+    }
+
+    assert(buf[0] == test_objs[0]);  // unchanged
+    assert(buf[1] == test_objs[1]);  // unchanged
+    assert(buf[2] == test_objs[0]);
+    assert(buf[3] == test_objs[1]);
+    assert(buf[4] == test_objs[2]);
+
+    Py_DECREF(a);
+    Py_RETURN_NONE;
+}
+
+// dest == src: backward copy where every write is a no-op.
+// buf = [0,1,2,3,4], copy src=&buf[1] n=3 to dest=&buf[1]
+// Expected result: buf unchanged.
+static PyObject *
+test_memmove_dest_eq_src(PyObject *self, PyObject *Py_UNUSED(arg))
+{
+    setup_test_objs();
+
+    PyObject *buf[NPTRS];
+    fill_buf(buf);
+
+    PyObject *a = call_memmove(&buf[1], &buf[1], 3);
+    if (a == NULL) {
+        return NULL;
+    }
+
+    for (int i = 0; i < NPTRS; i++) {
+        assert(buf[i] == test_objs[i]);
+    }
+
+    Py_DECREF(a);
+    Py_RETURN_NONE;
+}
+
+// Overlapping ranges, dest < src: forward copy is safe.
+// buf = [0,1,2,3,4], copy src=&buf[2] n=3 to dest=&buf[1]
+// Forward: buf[1]=buf[2]=2, buf[2]=buf[3]=3, buf[3]=buf[4]=4
+// Expected result: buf = [0,2,3,4,4]
+static PyObject *
+test_memmove_overlapping(PyObject *self, PyObject *Py_UNUSED(arg))
+{
+    setup_test_objs();
+
+    PyObject *buf[NPTRS];
+    fill_buf(buf);
+
+    PyObject *a = call_memmove(&buf[1], &buf[2], 3);
+    if (a == NULL) {
+        return NULL;
+    }
+
+    assert(buf[0] == test_objs[0]);  // unchanged
+    assert(buf[1] == test_objs[2]);
+    assert(buf[2] == test_objs[3]);
+    assert(buf[3] == test_objs[4]);
+    assert(buf[4] == test_objs[4]);  // unchanged
+
+    Py_DECREF(a);
+    Py_RETURN_NONE;
+}
+
+// Single owner fast path (GIL_DISABLED only): object owned by this thread
+// and not GC-shared => memmove is used instead of atomic stores.
+// In non-GIL builds this always takes the memmove path, so the test
+// degenerates to a basic correctness check.
+static PyObject *
+test_memmove_single_owner(PyObject *self, PyObject *Py_UNUSED(arg))
+{
+    setup_test_objs();
+
+    PyObject *a = PyList_New(0);
+    if (a == NULL) {
+        return NULL;
+    }
+
+#ifdef Py_GIL_DISABLED
+    // A freelist-reused list may carry the SHARED bit set by a sibling test.
+    // Clear it explicitly so this test exercises the single-owner fast path.
+    _PyObject_CLEAR_GC_BITS(a, _PyGC_BITS_SHARED);
+    assert(_Py_IsOwnedByCurrentThread(a) && !_PyObject_GC_IS_SHARED(a));
+#endif
+
+    PyObject *buf[NPTRS];
+    fill_buf(buf);
+
+    _Py_ptr_wise_atomic_memmove(a, &buf[0], &buf[2], 3);
+
+    assert(buf[0] == test_objs[2]);
+    assert(buf[1] == test_objs[3]);
+    assert(buf[2] == test_objs[4]);
+
+    Py_DECREF(a);
+    Py_RETURN_NONE;
+}
+
+
+static PyMethodDef test_methods[] = {
+    {"test_memmove_dest_lt_src",  test_memmove_dest_lt_src,  METH_NOARGS},
+    {"test_memmove_dest_gt_src",  test_memmove_dest_gt_src,  METH_NOARGS},
+    {"test_memmove_dest_eq_src",  test_memmove_dest_eq_src,  METH_NOARGS},
+    {"test_memmove_overlapping",  test_memmove_overlapping,  METH_NOARGS},
+    {"test_memmove_single_owner", test_memmove_single_owner, METH_NOARGS},
+    {NULL},
+};
+
+int
+_PyTestInternalCapi_Init_PtrWiseMemmove(PyObject *mod)
+{
+    return PyModule_AddFunctions(mod, test_methods);
+}