Drain each thread's page list during GC

Free pages awaiting QSBR are added to each thread's `mimalloc.page_list`.
The list is only drained if the owning thread needs new pages, performs
an allocation larger than mimalloc's medium-object limit, or the thread
exits. This means that we don't free pages if a thread's allocation
pattern quiesces or falls into a pattern that can be serviced by
existing pages. This can be problematic for threads that allocate a
bunch of memory and then settle into the previous pattern.

Author: Matt Page

Include/internal/pycore_pymem.h

@@ -129,6 +129,10 @@ extern void _PyMem_AbandonDelayed(PyThreadState *tstate);
 // On interpreter shutdown, frees all delayed free requests.
 extern void _PyMem_FiniDelayed(PyInterpreterState *interp);
 
+// Free every empty page parked on each thread's page_list that has reached its
+// QSBR goal
+extern void _PyMem_FreeParkedQSBRPages(PyInterpreterState *interp);
+
 #ifdef __cplusplus
 }
 #endif

Objects/obmalloc.c

@@ -226,16 +226,26 @@ _PyMem_mi_page_reclaimed(mi_page_t *page)
 #endif
 }
 
+#ifdef Py_GIL_DISABLED
+// Free the empty pages on `target`'s QSBR page list whose QSBR goal has been
+// reached.
+//
+// `cur` must be the calling thread's state: _Py_qsbr_poll() must be called
+// with the caller's own QSBR state. The goals themselves are global, so `cur`
+// can check pages parked by any thread.
+//
+// `target` is the thread whose page list is drained. It must be `cur`, or a
+// thread that is suspended (e.g., during a stop-the-world pause), because
+// _mi_page_free() modifies the heap and segment state of the thread that owns
+// the page.
 static void
-_PyMem_mi_heap_collect_qsbr(mi_heap_t *heap)
+_PyMem_mi_collect_tstate_page_list(_PyThreadStateImpl *cur, _PyThreadStateImpl *target)
 {
-#ifdef Py_GIL_DISABLED
-    if (!heap->page_use_qsbr) {
+    if (!_Py_atomic_load_int(&target->mimalloc.initialized)) {
+        // The thread may not have called tstate_mimalloc_bind() yet.
         return;
     }
-
-    _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
-    struct llist_node *head = &tstate->mimalloc.page_list;
+    struct llist_node *head = &target->mimalloc.page_list;
     if (llist_empty(head)) {
         return;
     }
@@ -249,13 +259,26 @@ _PyMem_mi_heap_collect_qsbr(mi_heap_t *heap)
             continue;
         }
 
-        if (!_Py_qsbr_poll(tstate->qsbr, page->qsbr_goal)) {
+        if (!_Py_qsbr_poll(cur->qsbr, page->qsbr_goal)) {
             return;
         }
 
         _PyMem_mi_page_clear_qsbr(page);
         _mi_page_free(page, mi_page_queue_of(page), false);
     }
+}
+#endif
+
+static void
+_PyMem_mi_heap_collect_qsbr(mi_heap_t *heap)
+{
+#ifdef Py_GIL_DISABLED
+    if (!heap->page_use_qsbr) {
+        return;
+    }
+
+    _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+    _PyMem_mi_collect_tstate_page_list(tstate, tstate);
 #endif
 }
 
@@ -1702,6 +1725,19 @@ _PyMem_FiniDelayed(PyInterpreterState *interp)
     }
 }
 
+#ifdef Py_GIL_DISABLED
+void
+_PyMem_FreeParkedQSBRPages(PyInterpreterState *interp)
+{
+    _PyThreadStateImpl *tstate = (_PyThreadStateImpl *) _PyThreadState_GET();
+    _Py_FOR_EACH_TSTATE_BEGIN(interp, p) {
+        _PyMem_mi_collect_tstate_page_list(tstate, (_PyThreadStateImpl *) p);
+    }
+    _Py_FOR_EACH_TSTATE_END(interp);
+}
+#endif
+
+
 /**************************/
 /* the "object" allocator */
 /**************************/

Python/gc_free_threading.c

@@ -2110,6 +2110,8 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state,
 
     process_delayed_frees(interp, state);
 
+    _PyMem_FreeParkedQSBRPages(interp);
+
     #ifdef GC_ENABLE_MARK_ALIVE
     // If gc.freeze() was used, it seems likely that doing this "mark alive"
     // pass will not be a performance win.  Typically the majority of alive