Repository navigation
gh-160303: Fix bytearray state after a failed del b[:n] #160304
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| Fix :class:`bytearray` reading and writing memory outside of its buffer | ||
| after ``del b[:n]`` or ``b[:n] = shorter`` failed with :exc:`MemoryError`. | ||
| The bytes are now removed even if shrinking the memory block fails; | ||
| :exc:`MemoryError` is still raised. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -266,37 +266,27 @@ bytearray_resize_storage(PyByteArrayObject *self, | |
| } | ||
|
|
||
|
|
||
| /* Resize the storage for requested_size bytes, even if it is already the | ||
| size of the bytearray. The caller must have checked that the bytearray | ||
| can be resized. */ | ||
| static int | ||
| bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) | ||
| bytearray_resize_unchecked(PyByteArrayObject *obj, Py_ssize_t requested_size) | ||
| { | ||
| _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); | ||
| PyByteArrayObject *obj = ((PyByteArrayObject *)self); | ||
| _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); | ||
| PyObject *self = (PyObject *)obj; | ||
|
|
||
| assert(obj->ob_bytes_object != NULL); | ||
| assert(requested_size >= 0); | ||
| assert(obj->ob_exports == 0); | ||
|
|
||
| /* All computations are done unsigned to avoid integer overflows | ||
| (see issue #22335). */ | ||
| size_t alloc = (size_t) obj->ob_alloc; | ||
| size_t logical_offset = (size_t) (obj->ob_start - obj->ob_bytes); | ||
| size_t size = (size_t) requested_size; | ||
|
|
||
| assert(self != NULL); | ||
| assert(PyByteArray_Check(self)); | ||
| assert(logical_offset <= alloc); | ||
|
|
||
| if (requested_size < 0) { | ||
| PyErr_Format(PyExc_ValueError, | ||
| "Can only resize to positive sizes, got %zd", requested_size); | ||
| return -1; | ||
| } | ||
|
|
||
| if (requested_size == Py_SIZE(self)) { | ||
| return 0; | ||
| } | ||
| if (!_canresize(obj)) { | ||
| return -1; | ||
| } | ||
|
|
||
| /* Resize to 0 resets to empty bytes (see issue #153419). */ | ||
| if (requested_size == 0) { | ||
| Py_SETREF(obj->ob_bytes_object, | ||
|
|
@@ -348,6 +338,30 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) | |
| return 0; | ||
| } | ||
|
|
||
| static int | ||
| bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) | ||
| { | ||
| _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); | ||
| PyByteArrayObject *obj = ((PyByteArrayObject *)self); | ||
|
|
||
| assert(self != NULL); | ||
| assert(PyByteArray_Check(self)); | ||
|
|
||
| if (requested_size < 0) { | ||
| PyErr_Format(PyExc_ValueError, | ||
| "Can only resize to positive sizes, got %zd", requested_size); | ||
| return -1; | ||
| } | ||
|
|
||
| if (requested_size == Py_SIZE(self)) { | ||
| return 0; | ||
| } | ||
| if (!_canresize(obj)) { | ||
| return -1; | ||
| } | ||
| return bytearray_resize_unchecked(obj, requested_size); | ||
| } | ||
|
|
||
| int | ||
| PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) | ||
| { | ||
|
|
@@ -612,8 +626,12 @@ bytearray_setslice_linear(PyByteArrayObject *self, | |
| return -1; | ||
|
|
||
| if (lo == 0) { | ||
| /* Shrink the buffer by advancing its logical start */ | ||
| /* Shrink the buffer by advancing its logical start. The bytes | ||
| are removed at this point; the resize below only releases | ||
| memory. */ | ||
| self->ob_start -= growth; | ||
| Py_SET_SIZE(self, Py_SIZE(self) + growth); | ||
| growth = 0; | ||
| /* | ||
| 0 lo hi old_size | ||
| | |<----avail----->|<-----tail------>| | ||
|
|
@@ -631,24 +649,16 @@ bytearray_setslice_linear(PyByteArrayObject *self, | |
| memmove(buf + lo + bytes_len, buf + hi, | ||
| Py_SIZE(self) - hi); | ||
| } | ||
| if (bytearray_resize_lock_held((PyObject *)self, | ||
| Py_SIZE(self) + growth) < 0) { | ||
| if (bytearray_resize_unchecked(self, Py_SIZE(self) + growth) < 0) { | ||
| /* Issue #19578: Handling the memory allocation failure here is | ||
| tricky here because the bytearray object has already been | ||
| modified. Depending on growth and lo, the behaviour is | ||
| different. | ||
|
|
||
| If growth < 0 and lo != 0, the operation is completed, but a | ||
| MemoryError is still raised and the memory block is not | ||
| shrunk. Otherwise, the bytearray is restored in its previous | ||
| state and a MemoryError is raised. */ | ||
| if (lo == 0) { | ||
| self->ob_start += growth; | ||
| return -1; | ||
| } | ||
| /* memmove() removed bytes, the bytearray object cannot be | ||
| restored in its previous state. */ | ||
| modified: the bytes are removed and the bytearray object | ||
| cannot be restored in its previous state. | ||
|
|
||
| The operation is completed, but a MemoryError is still | ||
| raised and the memory block is not shrunk. */ | ||
| Py_SET_SIZE(self, Py_SIZE(self) + growth); | ||
| bytearray_write_trailing_null_byte(self); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. From my claude:
I think we need a new remove from end test? (For most cases |
||
| res = -1; | ||
| } | ||
| buf = PyByteArray_AS_STRING(self); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When I'd use
bytearray_resize_uncheckedvs.bytearray_resize_lock_helddoesn't make a lot of sense to me... all these checks seem like they're still valid to do for the "bytearray_setslice_linear" case which now skips them withbytearray_resize_unchecked