bpo-20082: fix misbehavior of buffered writes to raw files in append mode

Author: embray

Lib/test/test_io.py

@@ -2396,6 +2396,31 @@ def test_interleaved_readline_write(self):
                 f.flush()
                 self.assertEqual(raw.getvalue(), b'1b\n2def\n3\n')
 
+    def test_append_write(self):
+        # Uses a real FileIO so that append behavior is reproduced accurately
+        with self.FileIO(os_helper.TESTFN, 'wb') as f:
+            f.write(b'test test')
+
+        with self.FileIO(os_helper.TESTFN, 'ab+') as raw:
+            with self.tp(raw) as f:
+                self.assertEqual(f.tell(), 9)
+                f.write(b'A')
+                f.seek(0)
+                self.assertEqual(f.read(), b'test testA')
+                f.seek(0)
+                self.assertEqual(f.read(1), b't')
+                self.assertEqual(f.write(b'B'), 1)
+                f.seek(0)
+
+                # This read previously returned b'tBst testA' but that is
+                # inccorect if the underlying raw file is in append mode;
+                # see bpo-20082
+                self.assertEqual(f.read(), b'test testAB')
+                f.flush()
+                f.seek(0)
+                self.assertEqual(f.read(), b'test testAB')
+
+
     # You can't construct a BufferedRandom over a non-seekable stream.
     test_unseekable = None
 

Modules/_io/bufferedio.c

@@ -204,6 +204,7 @@ typedef struct {
     int detached;
     int readable;
     int writable;
+    int appending;
     char finalizing;
 
     /* True if this is a vanilla Buffered object (rather than a user derived
@@ -1204,6 +1205,33 @@ buffered_tell(buffered *self, PyObject *Py_UNUSED(ignored))
     return PyLong_FromOff_t(pos);
 }
 
+static PyObject *
+_buffered_seek_unlocked(buffered *self, Py_off_t target, int whence)
+{
+    Py_off_t n;
+    PyObject *res = NULL;
+
+    if (self->writable) {
+        res = _bufferedwriter_flush_unlocked(self);
+        if (res == NULL)
+            return res;
+        Py_CLEAR(res);
+    }
+
+    /* TODO: align on block boundary and read buffer if needed? */
+    if (whence == 1)
+        target -= RAW_OFFSET(self);
+    n = _buffered_raw_seek(self, target, whence);
+    if (n == -1)
+        return res;
+    self->raw_pos = -1;
+    res = PyLong_FromOff_t(n);
+    if (res != NULL && self->readable)
+        _bufferedreader_reset_buf(self);
+
+    return res;
+}
+
 /*[clinic input]
 _io._Buffered.seek
     target as targetobj: object
@@ -1215,7 +1243,7 @@ static PyObject *
 _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence)
 /*[clinic end generated code: output=7ae0e8dc46efdefb input=a9c4920bfcba6163]*/
 {
-    Py_off_t target, n;
+    Py_off_t target;
     PyObject *res = NULL;
 
     CHECK_INITIALIZED(self)
@@ -1276,25 +1304,8 @@ _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence)
         return NULL;
 
     /* Fallback: invoke raw seek() method and clear buffer */
-    if (self->writable) {
-        res = _bufferedwriter_flush_unlocked(self);
-        if (res == NULL)
-            goto end;
-        Py_CLEAR(res);
-    }
+    res = _buffered_seek_unlocked(self, target, whence);
 
-    /* TODO: align on block boundary and read buffer if needed? */
-    if (whence == 1)
-        target -= RAW_OFFSET(self);
-    n = _buffered_raw_seek(self, target, whence);
-    if (n == -1)
-        goto end;
-    self->raw_pos = -1;
-    res = PyLong_FromOff_t(n);
-    if (res != NULL && self->readable)
-        _bufferedreader_reset_buf(self);
-
-end:
     LEAVE_BUFFERED(self)
     return res;
 }
@@ -1767,6 +1778,27 @@ _bufferedwriter_reset_buf(buffered *self)
     self->write_end = -1;
 }
 
+static void
+_bufferedwriter_set_append(buffered *self)
+{
+    PyObject *mode;
+
+    mode = _PyObject_GetAttrId(self->raw, &PyId_mode);
+    if (mode != NULL) {
+        /* Raw fileobj has no mode string so as far as we can know it has
+           normal write behavior */
+        if (PyUnicode_FindChar(mode, 'a', 0, PyUnicode_GET_LENGTH(mode), 1) != -1) {
+            self->appending = 1;
+        } else {
+            self->appending = 0;
+        }
+        Py_DECREF(mode);
+    } else {
+        PyErr_Clear();
+        self->appending = 0;
+    }
+}
+
 /*[clinic input]
 _io.BufferedWriter.__init__
     raw: object
@@ -1795,6 +1827,8 @@ _io_BufferedWriter___init___impl(buffered *self, PyObject *raw,
     self->readable = 0;
     self->writable = 1;
 
+    _bufferedwriter_set_append(self);
+
     self->buffer_size = buffer_size;
     if (_buffered_init(self) < 0)
         return -1;
@@ -1944,6 +1978,14 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer)
         self->pos = 0;
         self->raw_pos = 0;
     }
+
+    if (self->appending) {
+        res = _buffered_seek_unlocked(self, 0, SEEK_END);
+        if (res == NULL)
+            goto error;
+        Py_DECREF(res);
+    }
+
     avail = Py_SAFE_DOWNCAST(self->buffer_size - self->pos, Py_off_t, Py_ssize_t);
     if (buffer->len <= avail) {
         memcpy(self->buffer + self->pos, buffer->buf, buffer->len);
@@ -2309,6 +2351,8 @@ _io_BufferedRandom___init___impl(buffered *self, PyObject *raw,
     self->readable = 1;
     self->writable = 1;
 
+    _bufferedwriter_set_append(self);
+
     if (_buffered_init(self) < 0)
         return -1;
     _bufferedreader_reset_buf(self);