Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Doc/library/asyncio-protocol.rst
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,9 @@ Write-only Transports
This method does not block; it buffers the data and arranges for it
to be sent out asynchronously.

The transport keeps a reference to *data* rather than copying. *data*
must not be mutated while being sent.

.. method:: WriteTransport.writelines(list_of_data)

Write a list (or any iterable) of data bytes to the transport.
Expand Down
111 changes: 29 additions & 82 deletions Lib/_pydatetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -663,14 +663,6 @@ class timedelta:

def __new__(cls, days=0, seconds=0, microseconds=0,
milliseconds=0, minutes=0, hours=0, weeks=0):
# Doing this efficiently and accurately in C is going to be difficult
# and error-prone, due to ubiquitous overflow possibilities, and that
# C double doesn't have enough bits of precision to represent
# microseconds over 10K years faithfully. The code here tries to make
# explicit where go-fast assumptions can be relied on, in order to
# guide the C implementation; it's way more convoluted than speed-
# ignoring auto-overflow-to-long idiomatic Python could be.

for name, value in (
("days", days),
("seconds", seconds),
Expand All @@ -685,80 +677,35 @@ def __new__(cls, days=0, seconds=0, microseconds=0,
f"unsupported type for timedelta {name} component: {type(value).__name__}"
)

# Final values, all integer.
# s and us fit in 32-bit signed ints; d isn't bounded.
d = s = us = 0

# Normalize everything to days, seconds, microseconds.
days += weeks*7
seconds += minutes*60 + hours*3600
microseconds += milliseconds*1000

# Get rid of all fractions, and normalize s and us.
# Take a deep breath <wink>.
if isinstance(days, float):
dayfrac, days = _math.modf(days)
daysecondsfrac, daysecondswhole = _math.modf(dayfrac * (24.*3600.))
assert daysecondswhole == int(daysecondswhole) # can't overflow
s = int(daysecondswhole)
assert days == int(days)
d = int(days)
else:
daysecondsfrac = 0.0
d = days
assert isinstance(daysecondsfrac, float)
assert abs(daysecondsfrac) <= 1.0
assert isinstance(d, int)
assert abs(s) <= 24 * 3600
# days isn't referenced again before redefinition

if isinstance(seconds, float):
secondsfrac, seconds = _math.modf(seconds)
assert seconds == int(seconds)
seconds = int(seconds)
secondsfrac += daysecondsfrac
assert abs(secondsfrac) <= 2.0
else:
secondsfrac = daysecondsfrac
# daysecondsfrac isn't referenced again
assert isinstance(secondsfrac, float)
assert abs(secondsfrac) <= 2.0

assert isinstance(seconds, int)
days, seconds = divmod(seconds, 24*3600)
d += days
s += int(seconds) # can't overflow
assert isinstance(s, int)
assert abs(s) <= 2 * 24 * 3600
# seconds isn't referenced again before redefinition

usdouble = secondsfrac * 1e6
assert abs(usdouble) < 2.1e6 # exact value not critical
# secondsfrac isn't referenced again

if isinstance(microseconds, float):
microseconds = round(microseconds + usdouble)
seconds, microseconds = divmod(microseconds, 1000000)
days, seconds = divmod(seconds, 24*3600)
d += days
s += seconds
else:
microseconds = int(microseconds)
seconds, microseconds = divmod(microseconds, 1000000)
days, seconds = divmod(seconds, 24*3600)
d += days
s += seconds
microseconds = round(microseconds + usdouble)
assert isinstance(s, int)
assert isinstance(microseconds, int)
assert abs(s) <= 3 * 24 * 3600
assert abs(microseconds) < 3.1e6

# Just a little bit of carrying possible for microseconds and seconds.
seconds, us = divmod(microseconds, 1000000)
s += seconds
days, s = divmod(s, 24*3600)
d += days
# Sum the exact number of microseconds, like the C implementation:
# the integer part of each float is converted exactly, and only the
# fractional microseconds left over are added up as a float and
# rounded (half to even) at the end.
us = 0
leftover = 0.0
for value, factor in (
(microseconds, 1),
(milliseconds, 1000),
(seconds, 1000000),
(minutes, 60*1000000),
(hours, 3600*1000000),
(days, 24*3600*1000000),
(weeks, 7*24*3600*1000000)
):
if isinstance(value, float):
frac, whole = _math.modf(value)
us += int(whole) * factor
frac, whole = _math.modf(frac * factor)
us += int(whole)
leftover += frac
else:
us += int(value * factor)
if leftover:
n, m = leftover.as_integer_ratio()
us = _divide_and_round(us * m + n, m)

s, us = divmod(us, 1000000)
d, s = divmod(s, 24*3600)

assert isinstance(d, int)
assert isinstance(s, int) and 0 <= s < 24*3600
Expand Down
4 changes: 3 additions & 1 deletion Lib/asyncio/selector_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -1099,7 +1099,9 @@ def write(self, data):
# Not all was written; register write handler.
self._add_writer(self._sock_fd, self._write_ready)

# Add it to the buffer.
# Add it to the buffer as a view, so partial sends don't copy it.
if not isinstance(data, memoryview):
data = memoryview(data)
self._buffer.append(data)
self._buffer_size += len(data)
self._maybe_pause_protocol()
Expand Down
17 changes: 16 additions & 1 deletion Lib/asyncio/sslproto.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,9 @@ def __init__(self, loop, app_protocol, sslcontext, waiter,
# App data write buffering
self._write_backlog = collections.deque()
self._write_buffer_size = 0
# How much of self._outgoing was produced by _do_write(): encrypted
# app data, plus anything else OpenSSL flushed along with it
self._outgoing_app_size = 0

self._waiter = waiter
self._loop = loop
Expand Down Expand Up @@ -720,6 +723,7 @@ def _write_appdata(self, list_of_data):
self._fatal_error(ex, 'Fatal error on SSL protocol')

def _do_write(self):
pending = self._outgoing.pending
try:
while self._write_backlog:
data = self._write_backlog[0]
Expand All @@ -733,11 +737,13 @@ def _do_write(self):
self._write_buffer_size -= data_len
except SSLAgainErrors:
pass
self._outgoing_app_size += self._outgoing.pending - pending
self._process_outgoing()

def _process_outgoing(self):
if not self._ssl_writing_paused:
data = self._outgoing.read()
self._outgoing_app_size = 0
if len(data):
self._transport.write(data)
self._control_app_writing()
Expand Down Expand Up @@ -904,7 +910,15 @@ def resume():
# Flow control for reads from SSL socket

def _control_ssl_reading(self):
size = self._get_read_buffer_size()
# Also count the records OpenSSL writes on its own while reading
# (alerts, KeyUpdate replies, ...) which the peer does not read:
# the app is not involved in producing them, so pausing it does
# not bound them; only pausing reads from the peer does. Whatever
# was produced while writing app data (_outgoing_app_size) is left
# out so that two peers which both write before they read cannot
# deadlock.
size = (self._get_read_buffer_size() +
self._outgoing.pending - self._outgoing_app_size)
if size >= self._incoming_high_water and not self._ssl_reading_paused:
self._ssl_reading_paused = True
self._transport.pause_reading()
Expand Down Expand Up @@ -937,6 +951,7 @@ def resume_writing(self):
assert self._ssl_writing_paused
self._ssl_writing_paused = False
self._process_outgoing()
self._control_ssl_reading()

def _fatal_error(self, exc, message='Fatal error on transport'):
if self._transport:
Expand Down
7 changes: 7 additions & 0 deletions Lib/test/datetimetester.py
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,13 @@ def test_constructor(self):
eq(td(seconds=0.001), td(milliseconds=1))
eq(td(milliseconds=0.001), td(microseconds=1))

# Large float args keep their sub-second part
eq(td(weeks=12345678.9), td(86419752, 25920, 225))
eq(td(hours=123456789.1), td(5144032, 75959, 999979))
eq(td(minutes=1234567890.1), td(857338, 70205, 999994))
eq(td(days=1.1, weeks=1000000.1), td(7000001, 69119, 999986))
eq(td(seconds=0.5, microseconds=2.0**60), td(13343998, 77407, 346976))

# Check type of args to constructor
ra(TypeError, lambda: td(weeks='1'))
ra(TypeError, lambda: td(days='1'))
Expand Down
23 changes: 16 additions & 7 deletions Lib/test/test_asyncio/test_selector_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -855,14 +855,23 @@ def test_write_partial_none(self):
self.assertEqual(list_to_buffer([b'data']), transport._buffer)

def test_write_tryagain(self):
self.sock.send.side_effect = BlockingIOError

data = b'data'
transport = self.socket_transport()
transport.write(data)

self.loop.assert_writer(7, transport._write_ready)
self.assertEqual(list_to_buffer([b'data']), transport._buffer)
for buf in data, bytearray(data), memoryview(data):
with self.subTest(type=type(buf).__name__):
self.sock.send.side_effect = BlockingIOError

transport = self.socket_transport()
transport.write(buf)

self.loop.assert_writer(7, transport._write_ready)
self.assertEqual(list_to_buffer([b'data']), transport._buffer)
# gh-159503: the data is buffered as a view, not a copy
self.assertIsInstance(transport._buffer[0], memoryview)
self.assertIs(transport._buffer[0].obj, memoryview(buf).obj)
if isinstance(buf, bytearray):
# The buffered view pins the bytearray against resizing.
with self.assertRaises(BufferError):
buf.append(0)

def test_write_sendmsg_no_data(self):
self.sock.sendmsg = mock.Mock()
Expand Down
40 changes: 40 additions & 0 deletions Lib/test/test_asyncio/test_sslproto.py
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,46 @@ def test_write_after_closing(self):
# should not raise
self.assertIsNone(transp.write(b'data'))

def test_pause_reading_if_ssl_replies_are_not_read(self):
# Output which OpenSSL produces on its own while reading (alerts,
# KeyUpdate, ...) must not pile up if the peer does not read it.
ssl_proto = self.ssl_protocol()
transport = self.connection_made(ssl_proto, do_handshake=mock.Mock())
reply = b'x' * (ssl_proto._incoming_high_water // 2)

def read(*args):
ssl_proto._outgoing.write(reply)
raise ssl.SSLWantReadError
ssl_proto._sslobj.read.side_effect = read

ssl_proto.pause_writing()
ssl_proto.buffer_updated(0)
transport.pause_reading.assert_not_called()
ssl_proto.buffer_updated(0)
transport.pause_reading.assert_called_once_with()
transport.resume_reading.assert_not_called()
transport.write.assert_not_called()

ssl_proto.resume_writing()
transport.write.assert_called_once_with(reply * 2)
transport.resume_reading.assert_called_once_with()

def test_no_pause_reading_if_app_data_is_not_read(self):
# Peers which both write a lot before they read must not deadlock.
ssl_proto = self.ssl_protocol()
transport = self.connection_made(ssl_proto, do_handshake=mock.Mock())
ssl_proto._sslobj.write.side_effect = ssl_proto._outgoing.write
data = b'x' * ssl_proto._incoming_high_water

ssl_proto.pause_writing()
ssl_proto._app_transport.write(data)
ssl_proto.buffer_updated(0)
transport.pause_reading.assert_not_called()

ssl_proto.resume_writing()
transport.write.assert_called_once_with(data)
transport.resume_reading.assert_not_called()


##############################################################################
# Start TLS Tests
Expand Down
16 changes: 16 additions & 0 deletions Lib/test/test_io/test_memoryio.py
Original file line number Diff line number Diff line change
Expand Up @@ -591,6 +591,10 @@ def test_issue141311(self):
self.assertEqual(sys.maxsize, memio.seek(sys.maxsize))
buf = bytearray(2)
self.assertEqual(0, memio.readinto(buf))
# Writing there must not overflow the buffer size.
memio.seek(sys.maxsize - 1)
self.assertRaises((MemoryError, OverflowError), memio.write, b"x")
self.assertEqual(memio.getvalue(), b"")

def test_peek(self):
buf = self.buftype("1234567890")
Expand Down Expand Up @@ -984,6 +988,12 @@ def __index__(self):
finally:
size.result.release()

# Sizes above 2**31 - 1 must work where C long is 32-bit.
if sys.maxsize > 2**31:
stream = self.ioclass(b'abc')
self.assertEqual(stream.truncate(2**31), 2**31)
self.assertEqual(stream.getvalue(), b'abc')

def test_readinto_overlapping_buffer(self):
with self.ioclass(b'abcd') as stream:
with stream.getbuffer()[1:] as target:
Expand Down Expand Up @@ -1150,6 +1160,12 @@ def __index__(self):
with self.assertRaises(ValueError):
stream.truncate(Size())

# Sizes above 2**31 - 1 must work where C long is 32-bit.
if sys.maxsize > 2**31:
stream = self.ioclass("abc")
self.assertEqual(stream.truncate(2**31), 2**31)
self.assertEqual(stream.getvalue(), "abc")

# XXX: For the Python version of io.StringIO, this is highly
# dependent on the encoding used for the underlying buffer.
def test_widechar(self):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix quadratic copying in :mod:`asyncio` selector socket transports on partial
sends. A :class:`bytearray` passed to :meth:`~asyncio.WriteTransport.write`
now stays locked against resizing until it has been sent.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix :class:`io.BytesIO` raising :exc:`SystemError` instead of
:exc:`OverflowError` when writing at a position close to
:data:`sys.maxsize`.
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix :class:`io.StringIO` implementation of :meth:`io.IOBase.truncate`
rejecting sizes larger than ``2**31 - 1`` on Windows.
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix :class:`io.BytesIO` implementation of :meth:`io.IOBase.truncate` rejecting
sizes larger than ``2**31 - 1`` on Windows.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix the pure-Python implementation of :class:`datetime.timedelta` losing
sub-second precision with large float arguments. It now gives the same
results as the C implementation.
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Fix unbounded memory growth in :mod:`asyncio` TLS connections when a peer
stops reading but keeps sending records that make OpenSSL generate its own
replies (such as TLS 1.2 renegotiation alerts). Reading from the peer is now
paused when this output accumulates, instead of only when application data
does.
5 changes: 4 additions & 1 deletion Modules/_io/bytesio.c
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,9 @@ resize_buffer_lock_held(bytesio *self, size_t size)
/* Major upsize; resize up to exact size */
alloc = size + 1;
}
if (alloc > PY_SSIZE_T_MAX) {
goto overflow;
}

if (SHARED_BUF(self)) {
if (unshare_buffer_lock_held(self, alloc) < 0)
Expand Down Expand Up @@ -755,7 +758,7 @@ _io_BytesIO_truncate_impl(bytesio *self, PyObject *size)
new_size = self->pos;
}
else {
new_size = PyLong_AsLong(size);
new_size = PyNumber_AsSsize_t(size, PyExc_OverflowError);
if (new_size == -1 && PyErr_Occurred()) {
return NULL;
}
Expand Down
2 changes: 1 addition & 1 deletion Modules/_io/stringio.c
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,7 @@ _io_StringIO_truncate_impl(stringio *self, PyObject *pos)
size = self->pos;
}
else {
size = PyLong_AsLong(pos);
size = PyNumber_AsSsize_t(pos, PyExc_OverflowError);
if (size == -1 && PyErr_Occurred()) {
return NULL;
}
Expand Down
2 changes: 1 addition & 1 deletion Platforms/emscripten/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ await createEmscriptenModule({

- Threading is disabled by default. The ``configure`` option
``--enable-wasm-pthreads`` adds compiler flag ``-pthread`` and
linker flags ``-sUSE_PTHREADS -sPROXY_TO_PTHREAD``.
linker flags ``-pthread -sPROXY_TO_PTHREAD``.
- pthread support requires WASM threads and SharedArrayBuffer (bulk memory).
The Node.JS runtime keeps a pool of web workers around. Each web worker
uses several file descriptors (eventfd, epoll, pipe).
Expand Down
Loading
Loading