gh-161065: Fix sqlite3 deadlock on connections shared between threads

SQLite calls Python callbacks with the connection mutex held, so never
block on that mutex while holding the GIL: enter it first, releasing
the GIL only if it is contended.

Author: gpshead

Lib/test/test_sqlite3/test_dbapi.py

@@ -26,6 +26,7 @@
 import sqlite3 as sqlite
 import subprocess
 import sys
+import textwrap
 import threading
 import unittest
 import urllib.parse
@@ -1734,6 +1735,115 @@ def run(con, err):
             t.join()
             self.assertEqual(len(err), 0, "\n".join(err))
 
+    @requires_subprocess()
+    @unittest.skipUnless(sqlite.threadsafety == 3,
+                         "requires SQLite in serialized threading mode")
+    def test_shared_connection_callback_deadlock(self):
+        # Threads sharing a connection must not deadlock when SQLite calls
+        # back into Python.  A deadlock would hang the whole process, so run
+        # this in a child process.
+        SCRIPT = textwrap.dedent("""
+            import itertools
+            import sqlite3
+            import sys
+            import threading
+
+            sys.setswitchinterval(0.00001)
+
+            class Count:
+                def __init__(self):
+                    self.n = 0
+                def step(self, x):
+                    self.n += 1
+                def finalize(self):
+                    return self.n
+
+            def connect():
+                cx = sqlite3.connect(":memory:", check_same_thread=False)
+                cx.execute("create table t(x)")
+                cx.executemany("insert into t values(?)",
+                               ((str(i),) for i in range(50)))
+                return cx
+
+            def query(cx, sql, n, errors):
+                # Use different SQL in each thread and for each iteration:
+                # the threads must not share a cached statement, and the
+                # authorizer is only called when a statement is prepared.
+                try:
+                    for i in range(n, n + 100):
+                        params = ("", 100) if i % 2 else [None, 100]
+                        cx.execute(f"{sql} limit ?+{i}", params).fetchall()
+                except Exception as exc:
+                    errors.append(exc)
+
+            def run(name, sql, setup, concurrent=None):
+                print(name, flush=True)
+                cx = connect()
+                setup(cx)
+                errors = []
+                threads = [threading.Thread(target=query,
+                                            args=(cx, sql, 1000, errors))]
+                if concurrent is None:
+                    threads.append(threading.Thread(
+                        target=query, args=(cx, sql, 2000, errors)))
+                for t in threads:
+                    t.start()
+                while concurrent is not None and threads[0].is_alive():
+                    concurrent(cx)
+                for t in threads:
+                    t.join()
+                cx.close()
+                if errors:
+                    raise errors[0]
+
+            def function(cx):
+                cx.create_function("f", 1, lambda x: x)
+            def collation(cx):
+                cx.create_collation("c", lambda a, b: (a > b) - (a < b))
+
+            run("function", "select f(x) from t where x is not ?", function)
+            run("aggregate", "select agg(x) from t where x is not ? group by x",
+                lambda cx: cx.create_aggregate("agg", 1, Count))
+            run("collation",
+                "select x from t where x is not ? order by x collate c",
+                collation)
+            run("authorizer", "select x from t where x is not ?",
+                lambda cx: cx.set_authorizer(lambda *a: sqlite3.SQLITE_OK))
+            run("progress handler", "select x from t where x is not ?",
+                lambda cx: cx.set_progress_handler(lambda: 0, 1))
+            run("trace callback", "select x from t where x is not ?",
+                lambda cx: cx.set_trace_callback(lambda stmt: None))
+
+            # Register callbacks while another thread is in a callback.
+            # Use new names: existing functions and collations cannot be
+            # modified while statements are running.
+            sql = "select f(x) from t where x is not ?"
+            names = map("name{}".format, itertools.count())
+            run("create_function", sql, function,
+                lambda cx: cx.create_function(next(names), 0, lambda: None))
+            run("create_aggregate", sql, function,
+                lambda cx: cx.create_aggregate(next(names), 1, Count))
+            run("create_collation", sql, function,
+                lambda cx: cx.create_collation(next(names), lambda a, b: 0))
+            run("set_authorizer", sql, function,
+                lambda cx: cx.set_authorizer(None))
+            run("set_progress_handler", sql, function,
+                lambda cx: cx.set_progress_handler(None, 0))
+            run("set_trace_callback", sql, function,
+                lambda cx: cx.set_trace_callback(None))
+            run("setconfig", sql, function,
+                lambda cx: cx.setconfig(sqlite3.SQLITE_DBCONFIG_ENABLE_FKEY))
+            print("done")
+        """)
+        try:
+            proc = subprocess.run([sys.executable, "-c", SCRIPT],
+                                  capture_output=True, text=True,
+                                  timeout=SHORT_TIMEOUT)
+        except subprocess.TimeoutExpired as exc:
+            self.fail(f"deadlock; output: {exc.stdout!r}")
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        self.assertEndsWith(proc.stdout, "done\n")
+
 
 class ConstructorTests(unittest.TestCase):
     def test_date(self):

Misc/NEWS.d/next/Library/2026-10-22-06-39-13.gh-issue-161065.q7GdMx.rst

@@ -0,0 +1,4 @@
+Fix a deadlock in :mod:`sqlite3` when a connection created with
+``check_same_thread=False`` is used from several threads at once and has a
+Python callback registered, such as a user-defined function, aggregate,
+collation, authorizer, progress handler or trace callback.

Modules/_sqlite/connection.c

@@ -434,13 +434,15 @@ remove_callbacks(sqlite3 *db)
     /* None of these APIs can fail, as long as they are given a valid
      * database pointer. */
     assert(db != NULL);
+    pysqlite_db_mutex_enter(db);
     int rc = sqlite3_trace_v2(db, SQLITE_TRACE_STMT, 0, 0);
     assert(rc == SQLITE_OK), (void)rc;
 
     sqlite3_progress_handler(db, 0, 0, (void *)0);
 
     rc = sqlite3_set_authorizer(db, NULL, NULL);
     assert(rc == SQLITE_OK), (void)rc;
+    pysqlite_db_mutex_leave(db);
 }
 
 static int
@@ -1231,11 +1233,13 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self,
     if (ctx == NULL) {
         return NULL;
     }
+    pysqlite_db_mutex_enter(self->db);
     rc = sqlite3_create_function_v2(self->db, name, narg, flags, ctx,
                                     func_callback,
                                     NULL,
                                     NULL,
                                     &destructor_callback);  // will free 'ctx'
+    pysqlite_db_mutex_leave(self->db);
 
     if (rc != SQLITE_OK) {
         /* Workaround for SQLite bug: no error code or string is available here */
@@ -1376,21 +1380,25 @@ create_window_function_impl(pysqlite_Connection *self, PyTypeObject *cls,
     int flags = SQLITE_UTF8;
     int rc;
     if (Py_IsNone(aggregate_class)) {
+        pysqlite_db_mutex_enter(self->db);
         rc = sqlite3_create_window_function(self->db, name, num_params, flags,
                                             0, 0, 0, 0, 0, 0);
+        pysqlite_db_mutex_leave(self->db);
     }
     else {
         callback_context *ctx = create_callback_context(cls, aggregate_class);
         if (ctx == NULL) {
             return NULL;
         }
+        pysqlite_db_mutex_enter(self->db);
         rc = sqlite3_create_window_function(self->db, name, num_params, flags,
                                             ctx,
                                             &step_callback,
                                             &final_callback,
                                             &value_callback,
                                             &inverse_callback,
                                             &destructor_callback);
+        pysqlite_db_mutex_leave(self->db);
     }
 
     if (rc != SQLITE_OK) {
@@ -1435,11 +1443,13 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self,
     if (ctx == NULL) {
         return NULL;
     }
+    pysqlite_db_mutex_enter(self->db);
     rc = sqlite3_create_function_v2(self->db, name, n_arg, SQLITE_UTF8, ctx,
                                     0,
                                     &step_callback,
                                     &final_callback,
                                     &destructor_callback); // will free 'ctx'
+    pysqlite_db_mutex_leave(self->db);
     if (rc != SQLITE_OK) {
         /* Workaround for SQLite bug: no error code or string is available here */
         PyErr_SetString(self->OperationalError, "Error creating aggregate");
@@ -1593,15 +1603,19 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self,
 
     int rc;
     if (callable == Py_None) {
+        pysqlite_db_mutex_enter(self->db);
         rc = sqlite3_set_authorizer(self->db, NULL, NULL);
+        pysqlite_db_mutex_leave(self->db);
         set_callback_context(&self->authorizer_ctx, NULL);
     }
     else {
         callback_context *ctx = create_callback_context(cls, callable);
         if (ctx == NULL) {
             return NULL;
         }
+        pysqlite_db_mutex_enter(self->db);
         rc = sqlite3_set_authorizer(self->db, authorizer_callback, ctx);
+        pysqlite_db_mutex_leave(self->db);
         set_callback_context(&self->authorizer_ctx, ctx);
     }
     if (rc != SQLITE_OK) {
@@ -1644,15 +1658,19 @@ pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self,
 
     if (callable == Py_None) {
         /* None clears the progress handler previously set */
+        pysqlite_db_mutex_enter(self->db);
         sqlite3_progress_handler(self->db, 0, 0, (void*)0);
+        pysqlite_db_mutex_leave(self->db);
         set_callback_context(&self->progress_ctx, NULL);
     }
     else {
         callback_context *ctx = create_callback_context(cls, callable);
         if (ctx == NULL) {
             return NULL;
         }
+        pysqlite_db_mutex_enter(self->db);
         sqlite3_progress_handler(self->db, n, progress_callback, ctx);
+        pysqlite_db_mutex_leave(self->db);
         set_callback_context(&self->progress_ctx, ctx);
     }
     Py_RETURN_NONE;
@@ -1687,15 +1705,19 @@ pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self,
          * - https://sqlite.org/c3ref/c_trace.html
          * - https://sqlite.org/c3ref/trace_v2.html
          */
+        pysqlite_db_mutex_enter(self->db);
         sqlite3_trace_v2(self->db, SQLITE_TRACE_STMT, 0, 0);
+        pysqlite_db_mutex_leave(self->db);
         set_callback_context(&self->trace_ctx, NULL);
     }
     else {
         callback_context *ctx = create_callback_context(cls, callable);
         if (ctx == NULL) {
             return NULL;
         }
+        pysqlite_db_mutex_enter(self->db);
         sqlite3_trace_v2(self->db, SQLITE_TRACE_STMT, trace_callback, ctx);
+        pysqlite_db_mutex_leave(self->db);
         set_callback_context(&self->trace_ctx, ctx);
     }
 
@@ -1728,7 +1750,9 @@ pysqlite_connection_enable_load_extension_impl(pysqlite_Connection *self,
         return NULL;
     }
 
+    pysqlite_db_mutex_enter(self->db);
     rc = sqlite3_enable_load_extension(self->db, onoff);
+    pysqlite_db_mutex_leave(self->db);
 
     if (rc != SQLITE_OK) {
         PyErr_SetString(self->OperationalError,
@@ -1767,7 +1791,9 @@ pysqlite_connection_load_extension_impl(pysqlite_Connection *self,
         return NULL;
     }
 
+    pysqlite_db_mutex_enter(self->db);
     rc = sqlite3_load_extension(self->db, extension_name, entrypoint, &errmsg);
+    pysqlite_db_mutex_leave(self->db);
     if (rc != 0) {
         PyErr_SetString(self->OperationalError, errmsg);
         return NULL;
@@ -2232,8 +2258,10 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self,
     int rc;
     int flags = SQLITE_UTF8;
     if (callable == Py_None) {
+        pysqlite_db_mutex_enter(self->db);
         rc = sqlite3_create_collation_v2(self->db, name, flags,
                                          NULL, NULL, NULL);
+        pysqlite_db_mutex_leave(self->db);
     }
     else {
         if (!PyCallable_Check(callable)) {
@@ -2244,9 +2272,11 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self,
         if (ctx == NULL) {
             return NULL;
         }
+        pysqlite_db_mutex_enter(self->db);
         rc = sqlite3_create_collation_v2(self->db, name, flags, ctx,
                                          &collation_callback,
                                          &destructor_callback);
+        pysqlite_db_mutex_leave(self->db);
     }
 
     if (rc != SQLITE_OK) {
@@ -2570,7 +2600,9 @@ setconfig_impl(pysqlite_Connection *self, int op, int enable)
     }
 
     int actual;
+    pysqlite_db_mutex_enter(self->db);
     int rc = sqlite3_db_config(self->db, op, enable, &actual);
+    pysqlite_db_mutex_leave(self->db);
     if (rc != SQLITE_OK) {
         set_error_from_db(self->state, self->db);
         return NULL;
@@ -2605,7 +2637,9 @@ getconfig_impl(pysqlite_Connection *self, int op)
     }
 
     int current;
+    pysqlite_db_mutex_enter(self->db);
     int rc = sqlite3_db_config(self->db, op, -1, &current);
+    pysqlite_db_mutex_leave(self->db);
     if (rc != SQLITE_OK) {
         set_error_from_db(self->state, self->db);
         return -1;