From 89a8ac91f07851c8cb554bb9a4b1126523616c44 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2026 21:22:34 +0200 Subject: [PATCH 1/3] gh-156939: Detect buffer overflow in PyBytesWriter in debug mode Reserve one byte in PyBytesWriter used as a canary byte: set it to a special value. PyBytesWriter_Finish() checks if the canary byte has been overriden. Add a test on the feature. Update buffer overflow check in fcntl: allocate extra guard bytes in the writer and then truncate these bytes. --- Lib/test/test_capi/test_bytes.py | 21 +++++++ ...-09-04-16-41-07.gh-issue-156939.bKaQuE.rst | 2 + Modules/_testcapi/bytes.c | 35 +++++++++++- Modules/fcntlmodule.c | 16 ++++-- Objects/bytesobject.c | 55 ++++++++++++++++++- 5 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-09-04-16-41-07.gh-issue-156939.bKaQuE.rst diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 4c1431bacef0a2..d997e578f635ea 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -1,5 +1,7 @@ +import textwrap import unittest from test.support import import_helper +from test.support.script_helper import assert_python_failure _testlimitedcapi = import_helper.import_module('_testlimitedcapi') _testcapi = import_helper.import_module('_testcapi') @@ -430,6 +432,25 @@ def test_example_resize(self): def test_example_highlevel(self): self.assertEqual(_testcapi.byteswriter_highlevel(), b'Hello World!') + def test_canary_byte(self): + small_buffer = _testcapi.PyBytesWriter_small_buffer + large_size = small_buffer * 10 + + # Test small buffer and large buffer + for size in (0, 3, large_size): + with self.subTest(size=size): + code = textwrap.dedent(f""" + from test.support import SuppressCrashReport + import _testcapi + size = {size} + data = b'x' * size + with SuppressCrashReport(): + _testcapi.byteswriter_test_canary_byte(data) + """) + proc = assert_python_failure('-c', code) + self.assertIn(b'Buffer overflow detected in PyBytesWriter', + proc.err) + class ByteArrayWriterTest(BaseWriterTest, unittest.TestCase): result_type = bytearray diff --git a/Misc/NEWS.d/next/C_API/2026-09-04-16-41-07.gh-issue-156939.bKaQuE.rst b/Misc/NEWS.d/next/C_API/2026-09-04-16-41-07.gh-issue-156939.bKaQuE.rst new file mode 100644 index 00000000000000..57c9a0ab46e904 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-04-16-41-07.gh-issue-156939.bKaQuE.rst @@ -0,0 +1,2 @@ +When Python is built in debug mode, :c:type:`PyBytesWriter` now detects +buffer overflow. Patch by Victor Stinner. diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index 4830cc8b54bd83..83d1dab6c749f9 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -151,7 +151,7 @@ writer_write_bytes(PyObject *self_raw, PyObject *args) return NULL; } - char *bytes; + const char *bytes; Py_ssize_t unused_size, size; if (!PyArg_ParseTuple(args, "y#n", &bytes, &unused_size, &size)) { return NULL; @@ -454,6 +454,38 @@ test_byteswriter_ptr(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } +// Trigger a buffer overflow on purpose to test the canary byte feature +// which detects buffer overflow +static PyObject * +byteswriter_test_canary_byte(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *str; + Py_ssize_t len; + if (!PyArg_ParseTuple(args, "s#", &str, &len)) { + return NULL; + } + + PyBytesWriter *writer = PyBytesWriter_Create(0); + if (writer == NULL) { + goto error; + } + if (PyBytesWriter_Grow(writer, len) < 0) { + goto error; + } + char *data = PyBytesWriter_GetData(writer); + if (len) { + memcpy(data, str, len); + } + data[len] = '#'; // Overflow! + + return PyBytesWriter_Finish(writer); + +error: + PyBytesWriter_Discard(writer); + return NULL; +} + + static PyMethodDef test_methods[] = { {"bytes_resize", bytes_resize, METH_VARARGS}, {"bytes_join", bytes_join, METH_VARARGS}, @@ -461,6 +493,7 @@ static PyMethodDef test_methods[] = { {"byteswriter_resize", byteswriter_resize, METH_NOARGS}, {"byteswriter_highlevel", byteswriter_highlevel, METH_NOARGS}, {"test_byteswriter_ptr", test_byteswriter_ptr, METH_NOARGS}, + {"byteswriter_test_canary_byte", byteswriter_test_canary_byte, METH_VARARGS}, {NULL}, }; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index e6a40ffc5a2614..5dd3df9bb408f0 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -121,13 +121,14 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) return PyBytes_FromStringAndSize(buf, len); } else { - PyBytesWriter *writer = PyBytesWriter_Create(len); + PyBytesWriter *writer = PyBytesWriter_Create(len + GUARDSZ); if (writer == NULL) { PyBuffer_Release(&view); return NULL; } char *ptr = PyBytesWriter_GetData(writer); memcpy(ptr, view.buf, len); + memcpy(ptr + len, guard, GUARDSZ); PyBuffer_Release(&view); do { @@ -142,7 +143,7 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) PyBytesWriter_Discard(writer); return NULL; } - if (ptr[len] != '\0') { + if (memcmp(ptr + len, guard, GUARDSZ) != 0) { PyErr_SetString(PyExc_SystemError, "Memory corruption in fcntl() due to " "buffer overflow. " @@ -151,7 +152,8 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) PyBytesWriter_Discard(writer); return NULL; } - return PyBytesWriter_Finish(writer); + // Truncate the trailing guard bytes + return PyBytesWriter_FinishWithSize(writer, len); } #undef FCNTL_BUFSZ } @@ -316,13 +318,14 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, return PyBytes_FromStringAndSize(buf, len); } else { - PyBytesWriter *writer = PyBytesWriter_Create(len); + PyBytesWriter *writer = PyBytesWriter_Create(len + GUARDSZ); if (writer == NULL) { PyBuffer_Release(&view); return NULL; } char *ptr = PyBytesWriter_GetData(writer); memcpy(ptr, view.buf, len); + memcpy(ptr + len, guard, GUARDSZ); PyBuffer_Release(&view); do { @@ -337,7 +340,7 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, PyBytesWriter_Discard(writer); return NULL; } - if (ptr[len] != '\0') { + if (memcmp(ptr + len, guard, GUARDSZ) != 0) { PyErr_SetString(PyExc_SystemError, "Memory corruption in ioctl() due to " "buffer overflow. " @@ -346,7 +349,8 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, PyBytesWriter_Discard(writer); return NULL; } - return PyBytesWriter_Finish(writer); + // Truncate the trailing guard bytes + return PyBytesWriter_FinishWithSize(writer, len); } #undef IOCTL_BUFSZ } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 2ae55b33f4f49d..a7b6b5a7df1e93 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3593,6 +3593,10 @@ _PyBytes_RepeatBuffer(char* dest, Py_ssize_t len_dest, // --- PyBytesWriter API ----------------------------------------------------- +// Use a value different than NUL (0) to be able to detect overflow writing +// one extra NUL byte which is a common error. +#define PyBytesWriter_CANARY_BYTE PYMEM_DEADBYTE + static inline char* byteswriter_data(PyBytesWriter *writer) { @@ -3604,7 +3608,8 @@ static inline Py_ssize_t byteswriter_allocated(PyBytesWriter *writer) { if (writer->obj == NULL) { - return sizeof(writer->small_buffer); + // Reserve the last byte for the canary byte + return sizeof(writer->small_buffer) - 1; } else if (writer->use_bytearray) { return PyByteArray_GET_SIZE(writer->obj); @@ -3615,6 +3620,31 @@ byteswriter_allocated(PyBytesWriter *writer) } +#ifdef Py_DEBUG +static void +byteswriter_check_canary_byte(PyBytesWriter *writer) +{ + const unsigned char *data = (const unsigned char*)byteswriter_data(writer); + unsigned char canary = data[writer->size]; + if (canary != PyBytesWriter_CANARY_BYTE) { + _Py_FatalErrorFormat(__func__, + "Buffer overflow detected in PyBytesWriter %p: " + "one byte written after the buffer " + "(at position %zd)", + writer, writer->size); + } +} + + +static void +byteswriter_write_canary_byte(PyBytesWriter *writer) +{ + unsigned char *data = (unsigned char*)byteswriter_data(writer); + data[writer->size] = PyBytesWriter_CANARY_BYTE; +} +#endif + + #ifdef MS_WINDOWS /* On Windows, overallocate by 50% is the best factor */ # define OVERALLOCATE_FACTOR 2 @@ -3719,6 +3749,7 @@ byteswriter_create(Py_ssize_t size, int use_bytearray) } #ifdef Py_DEBUG memset(byteswriter_data(writer), 0xff, byteswriter_allocated(writer)); + byteswriter_write_canary_byte(writer); #endif return writer; } @@ -3764,6 +3795,19 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) goto error; } +#ifdef Py_DEBUG + // Check for buffer overflow + byteswriter_check_canary_byte(writer); + + if (writer->obj != NULL) { + // byteswriter_write_canary_byte() can override the trailing NUL byte. + // So reset the trailing NUL byte to NUL. + Py_ssize_t allocated = byteswriter_allocated(writer); + char *data = byteswriter_data(writer); + data[allocated] = '\0'; + } +#endif + PyObject *result; if (size == 0) { result = bytes_get_empty(); @@ -3850,6 +3894,9 @@ PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) return -1; } writer->size = size; +#ifdef Py_DEBUG + byteswriter_write_canary_byte(writer); +#endif return 0; } @@ -3883,6 +3930,9 @@ PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t size) return -1; } writer->size = size; +#ifdef Py_DEBUG + byteswriter_write_canary_byte(writer); +#endif return 0; } @@ -3948,5 +3998,8 @@ _PyBytesWriter_ResizeToAllocated(PyBytesWriter *writer) { Py_ssize_t allocated = byteswriter_allocated(writer); writer->size = allocated; +#ifdef Py_DEBUG + byteswriter_write_canary_byte(writer); +#endif return allocated; } From 95e37016f1ac0792495b9a10ce413c271c9b6357 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2026 21:47:03 +0200 Subject: [PATCH 2/3] Skip test_canary_byte on release build --- Lib/test/test_capi/test_bytes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index d997e578f635ea..632881eb1b940e 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -1,5 +1,6 @@ import textwrap import unittest +from test import support from test.support import import_helper from test.support.script_helper import assert_python_failure @@ -432,6 +433,7 @@ def test_example_resize(self): def test_example_highlevel(self): self.assertEqual(_testcapi.byteswriter_highlevel(), b'Hello World!') + @unittest.skipUnless(support.Py_DEBUG, 'need a Python debug build') def test_canary_byte(self): small_buffer = _testcapi.PyBytesWriter_small_buffer large_size = small_buffer * 10 From 0dbd2cc2855e901dcc8995d2829320509233b22a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2026 21:53:37 +0200 Subject: [PATCH 3/3] Cleanup test_canary_byte() Avoid calling PyBytesWriter_Grow(). --- Lib/test/test_capi/test_bytes.py | 2 ++ Modules/_testcapi/bytes.c | 13 ++++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 632881eb1b940e..de53c9bc45670c 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -452,6 +452,8 @@ def test_canary_byte(self): proc = assert_python_failure('-c', code) self.assertIn(b'Buffer overflow detected in PyBytesWriter', proc.err) + self.assertIn(f'at position {size}'.encode(), + proc.err) class ByteArrayWriterTest(BaseWriterTest, unittest.TestCase): diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index 83d1dab6c749f9..b2c7a9ca464e5e 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -465,24 +465,19 @@ byteswriter_test_canary_byte(PyObject *Py_UNUSED(module), PyObject *args) return NULL; } - PyBytesWriter *writer = PyBytesWriter_Create(0); + PyBytesWriter *writer = PyBytesWriter_Create(len); if (writer == NULL) { - goto error; - } - if (PyBytesWriter_Grow(writer, len) < 0) { - goto error; + return NULL; } + char *data = PyBytesWriter_GetData(writer); if (len) { memcpy(data, str, len); } data[len] = '#'; // Overflow! + // In debug mode, PyBytesWriter_Finish() checks for buffer overflow return PyBytesWriter_Finish(writer); - -error: - PyBytesWriter_Discard(writer); - return NULL; }