Skip to content

Commit 28c388c

Browse files
committed
Merge remote-tracking branch 'upstream/main' into pr-145149
2 parents 6e80869 + 0387541 commit 28c388c

24 files changed

Lines changed: 276 additions & 101 deletions

‎Doc/library/asyncio-protocol.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,9 @@ Write-only Transports
333333
This method does not block; it buffers the data and arranges for it
334334
to be sent out asynchronously.
335335

336+
The transport keeps a reference to *data* rather than copying. *data*
337+
must not be mutated while being sent.
338+
336339
.. method:: WriteTransport.writelines(list_of_data)
337340

338341
Write a list (or any iterable) of data bytes to the transport.

‎Doc/library/socket.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1065,6 +1065,9 @@ The :mod:`!socket` module also offers various network-related services:
10651065
for IPv6 multicast addresses, string representing an address will not
10661066
contain ``%scope_id`` part.
10671067

1068+
.. versionchanged:: next
1069+
Null characters are now rejected in the *host* and *port* parameters.
1070+
10681071
.. _Happy Eyeballs: https://en.wikipedia.org/wiki/Happy_Eyeballs
10691072

10701073
.. function:: getfqdn([name])

‎Lib/_pydatetime.py‎

Lines changed: 29 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -663,14 +663,6 @@ class timedelta:
663663

664664
def __new__(cls, days=0, seconds=0, microseconds=0,
665665
milliseconds=0, minutes=0, hours=0, weeks=0):
666-
# Doing this efficiently and accurately in C is going to be difficult
667-
# and error-prone, due to ubiquitous overflow possibilities, and that
668-
# C double doesn't have enough bits of precision to represent
669-
# microseconds over 10K years faithfully. The code here tries to make
670-
# explicit where go-fast assumptions can be relied on, in order to
671-
# guide the C implementation; it's way more convoluted than speed-
672-
# ignoring auto-overflow-to-long idiomatic Python could be.
673-
674666
for name, value in (
675667
("days", days),
676668
("seconds", seconds),
@@ -685,80 +677,35 @@ def __new__(cls, days=0, seconds=0, microseconds=0,
685677
f"unsupported type for timedelta {name} component: {type(value).__name__}"
686678
)
687679

688-
# Final values, all integer.
689-
# s and us fit in 32-bit signed ints; d isn't bounded.
690-
d = s = us = 0
691-
692-
# Normalize everything to days, seconds, microseconds.
693-
days += weeks*7
694-
seconds += minutes*60 + hours*3600
695-
microseconds += milliseconds*1000
696-
697-
# Get rid of all fractions, and normalize s and us.
698-
# Take a deep breath <wink>.
699-
if isinstance(days, float):
700-
dayfrac, days = _math.modf(days)
701-
daysecondsfrac, daysecondswhole = _math.modf(dayfrac * (24.*3600.))
702-
assert daysecondswhole == int(daysecondswhole) # can't overflow
703-
s = int(daysecondswhole)
704-
assert days == int(days)
705-
d = int(days)
706-
else:
707-
daysecondsfrac = 0.0
708-
d = days
709-
assert isinstance(daysecondsfrac, float)
710-
assert abs(daysecondsfrac) <= 1.0
711-
assert isinstance(d, int)
712-
assert abs(s) <= 24 * 3600
713-
# days isn't referenced again before redefinition
714-
715-
if isinstance(seconds, float):
716-
secondsfrac, seconds = _math.modf(seconds)
717-
assert seconds == int(seconds)
718-
seconds = int(seconds)
719-
secondsfrac += daysecondsfrac
720-
assert abs(secondsfrac) <= 2.0
721-
else:
722-
secondsfrac = daysecondsfrac
723-
# daysecondsfrac isn't referenced again
724-
assert isinstance(secondsfrac, float)
725-
assert abs(secondsfrac) <= 2.0
726-
727-
assert isinstance(seconds, int)
728-
days, seconds = divmod(seconds, 24*3600)
729-
d += days
730-
s += int(seconds) # can't overflow
731-
assert isinstance(s, int)
732-
assert abs(s) <= 2 * 24 * 3600
733-
# seconds isn't referenced again before redefinition
734-
735-
usdouble = secondsfrac * 1e6
736-
assert abs(usdouble) < 2.1e6 # exact value not critical
737-
# secondsfrac isn't referenced again
738-
739-
if isinstance(microseconds, float):
740-
microseconds = round(microseconds + usdouble)
741-
seconds, microseconds = divmod(microseconds, 1000000)
742-
days, seconds = divmod(seconds, 24*3600)
743-
d += days
744-
s += seconds
745-
else:
746-
microseconds = int(microseconds)
747-
seconds, microseconds = divmod(microseconds, 1000000)
748-
days, seconds = divmod(seconds, 24*3600)
749-
d += days
750-
s += seconds
751-
microseconds = round(microseconds + usdouble)
752-
assert isinstance(s, int)
753-
assert isinstance(microseconds, int)
754-
assert abs(s) <= 3 * 24 * 3600
755-
assert abs(microseconds) < 3.1e6
756-
757-
# Just a little bit of carrying possible for microseconds and seconds.
758-
seconds, us = divmod(microseconds, 1000000)
759-
s += seconds
760-
days, s = divmod(s, 24*3600)
761-
d += days
680+
# Sum the exact number of microseconds, like the C implementation:
681+
# the integer part of each float is converted exactly, and only the
682+
# fractional microseconds left over are added up as a float and
683+
# rounded (half to even) at the end.
684+
us = 0
685+
leftover = 0.0
686+
for value, factor in (
687+
(microseconds, 1),
688+
(milliseconds, 1000),
689+
(seconds, 1000000),
690+
(minutes, 60*1000000),
691+
(hours, 3600*1000000),
692+
(days, 24*3600*1000000),
693+
(weeks, 7*24*3600*1000000)
694+
):
695+
if isinstance(value, float):
696+
frac, whole = _math.modf(value)
697+
us += int(whole) * factor
698+
frac, whole = _math.modf(frac * factor)
699+
us += int(whole)
700+
leftover += frac
701+
else:
702+
us += int(value * factor)
703+
if leftover:
704+
n, m = leftover.as_integer_ratio()
705+
us = _divide_and_round(us * m + n, m)
706+
707+
s, us = divmod(us, 1000000)
708+
d, s = divmod(s, 24*3600)
762709

763710
assert isinstance(d, int)
764711
assert isinstance(s, int) and 0 <= s < 24*3600

‎Lib/asyncio/selector_events.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1099,7 +1099,9 @@ def write(self, data):
10991099
# Not all was written; register write handler.
11001100
self._add_writer(self._sock_fd, self._write_ready)
11011101

1102-
# Add it to the buffer.
1102+
# Add it to the buffer as a view, so partial sends don't copy it.
1103+
if not isinstance(data, memoryview):
1104+
data = memoryview(data)
11031105
self._buffer.append(data)
11041106
self._buffer_size += len(data)
11051107
self._maybe_pause_protocol()

‎Lib/asyncio/sslproto.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,9 @@ def __init__(self, loop, app_protocol, sslcontext, waiter,
308308
# App data write buffering
309309
self._write_backlog = collections.deque()
310310
self._write_buffer_size = 0
311+
# How much of self._outgoing was produced by _do_write(): encrypted
312+
# app data, plus anything else OpenSSL flushed along with it
313+
self._outgoing_app_size = 0
311314

312315
self._waiter = waiter
313316
self._loop = loop
@@ -720,6 +723,7 @@ def _write_appdata(self, list_of_data):
720723
self._fatal_error(ex, 'Fatal error on SSL protocol')
721724

722725
def _do_write(self):
726+
pending = self._outgoing.pending
723727
try:
724728
while self._write_backlog:
725729
data = self._write_backlog[0]
@@ -733,11 +737,13 @@ def _do_write(self):
733737
self._write_buffer_size -= data_len
734738
except SSLAgainErrors:
735739
pass
740+
self._outgoing_app_size += self._outgoing.pending - pending
736741
self._process_outgoing()
737742

738743
def _process_outgoing(self):
739744
if not self._ssl_writing_paused:
740745
data = self._outgoing.read()
746+
self._outgoing_app_size = 0
741747
if len(data):
742748
self._transport.write(data)
743749
self._control_app_writing()
@@ -904,7 +910,15 @@ def resume():
904910
# Flow control for reads from SSL socket
905911

906912
def _control_ssl_reading(self):
907-
size = self._get_read_buffer_size()
913+
# Also count the records OpenSSL writes on its own while reading
914+
# (alerts, KeyUpdate replies, ...) which the peer does not read:
915+
# the app is not involved in producing them, so pausing it does
916+
# not bound them; only pausing reads from the peer does. Whatever
917+
# was produced while writing app data (_outgoing_app_size) is left
918+
# out so that two peers which both write before they read cannot
919+
# deadlock.
920+
size = (self._get_read_buffer_size() +
921+
self._outgoing.pending - self._outgoing_app_size)
908922
if size >= self._incoming_high_water and not self._ssl_reading_paused:
909923
self._ssl_reading_paused = True
910924
self._transport.pause_reading()
@@ -937,6 +951,7 @@ def resume_writing(self):
937951
assert self._ssl_writing_paused
938952
self._ssl_writing_paused = False
939953
self._process_outgoing()
954+
self._control_ssl_reading()
940955

941956
def _fatal_error(self, exc, message='Fatal error on transport'):
942957
if self._transport:

‎Lib/test/datetimetester.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,13 @@ def test_constructor(self):
554554
eq(td(seconds=0.001), td(milliseconds=1))
555555
eq(td(milliseconds=0.001), td(microseconds=1))
556556

557+
# Large float args keep their sub-second part
558+
eq(td(weeks=12345678.9), td(86419752, 25920, 225))
559+
eq(td(hours=123456789.1), td(5144032, 75959, 999979))
560+
eq(td(minutes=1234567890.1), td(857338, 70205, 999994))
561+
eq(td(days=1.1, weeks=1000000.1), td(7000001, 69119, 999986))
562+
eq(td(seconds=0.5, microseconds=2.0**60), td(13343998, 77407, 346976))
563+
557564
# Check type of args to constructor
558565
ra(TypeError, lambda: td(weeks='1'))
559566
ra(TypeError, lambda: td(days='1'))

‎Lib/test/test_asyncio/test_selector_events.py‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -855,14 +855,23 @@ def test_write_partial_none(self):
855855
self.assertEqual(list_to_buffer([b'data']), transport._buffer)
856856

857857
def test_write_tryagain(self):
858-
self.sock.send.side_effect = BlockingIOError
859-
860858
data = b'data'
861-
transport = self.socket_transport()
862-
transport.write(data)
863-
864-
self.loop.assert_writer(7, transport._write_ready)
865-
self.assertEqual(list_to_buffer([b'data']), transport._buffer)
859+
for buf in data, bytearray(data), memoryview(data):
860+
with self.subTest(type=type(buf).__name__):
861+
self.sock.send.side_effect = BlockingIOError
862+
863+
transport = self.socket_transport()
864+
transport.write(buf)
865+
866+
self.loop.assert_writer(7, transport._write_ready)
867+
self.assertEqual(list_to_buffer([b'data']), transport._buffer)
868+
# gh-159503: the data is buffered as a view, not a copy
869+
self.assertIsInstance(transport._buffer[0], memoryview)
870+
self.assertIs(transport._buffer[0].obj, memoryview(buf).obj)
871+
if isinstance(buf, bytearray):
872+
# The buffered view pins the bytearray against resizing.
873+
with self.assertRaises(BufferError):
874+
buf.append(0)
866875

867876
def test_write_sendmsg_no_data(self):
868877
self.sock.sendmsg = mock.Mock()

‎Lib/test/test_asyncio/test_sslproto.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,46 @@ def test_write_after_closing(self):
289289
# should not raise
290290
self.assertIsNone(transp.write(b'data'))
291291

292+
def test_pause_reading_if_ssl_replies_are_not_read(self):
293+
# Output which OpenSSL produces on its own while reading (alerts,
294+
# KeyUpdate, ...) must not pile up if the peer does not read it.
295+
ssl_proto = self.ssl_protocol()
296+
transport = self.connection_made(ssl_proto, do_handshake=mock.Mock())
297+
reply = b'x' * (ssl_proto._incoming_high_water // 2)
298+
299+
def read(*args):
300+
ssl_proto._outgoing.write(reply)
301+
raise ssl.SSLWantReadError
302+
ssl_proto._sslobj.read.side_effect = read
303+
304+
ssl_proto.pause_writing()
305+
ssl_proto.buffer_updated(0)
306+
transport.pause_reading.assert_not_called()
307+
ssl_proto.buffer_updated(0)
308+
transport.pause_reading.assert_called_once_with()
309+
transport.resume_reading.assert_not_called()
310+
transport.write.assert_not_called()
311+
312+
ssl_proto.resume_writing()
313+
transport.write.assert_called_once_with(reply * 2)
314+
transport.resume_reading.assert_called_once_with()
315+
316+
def test_no_pause_reading_if_app_data_is_not_read(self):
317+
# Peers which both write a lot before they read must not deadlock.
318+
ssl_proto = self.ssl_protocol()
319+
transport = self.connection_made(ssl_proto, do_handshake=mock.Mock())
320+
ssl_proto._sslobj.write.side_effect = ssl_proto._outgoing.write
321+
data = b'x' * ssl_proto._incoming_high_water
322+
323+
ssl_proto.pause_writing()
324+
ssl_proto._app_transport.write(data)
325+
ssl_proto.buffer_updated(0)
326+
transport.pause_reading.assert_not_called()
327+
328+
ssl_proto.resume_writing()
329+
transport.write.assert_called_once_with(data)
330+
transport.resume_reading.assert_not_called()
331+
292332

293333
##############################################################################
294334
# Start TLS Tests

‎Lib/test/test_asyncio/test_tasks.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,52 @@ async def notmuch():
416416
self.assertEqual(t.get_name(), '123456789')
417417
self.loop.run_until_complete(t)
418418

419+
# The free-threaded GC publishes allocation counts in batches, so the
420+
# collection cannot be scheduled to run inside get_name().
421+
@support.requires_gil_enabled("needs precise GC scheduling")
422+
def test_task_get_name_reentered_during_formatting(self):
423+
# gh-158159: formatting the lazy default name can run the GC, and a
424+
# finalizer may call get_name() on the same task, replacing the name
425+
# the outer call is still formatting.
426+
loop = self.loop
427+
observed = {}
428+
keep = []
429+
task = None
430+
431+
class Reenter:
432+
def __init__(self):
433+
self.cycle = self
434+
435+
def __del__(self):
436+
observed['inner'] = task.get_name()
437+
# Reuse the memory of a counter that was freed too early.
438+
keep.extend(10**9 + i for i in range(8))
439+
440+
async def notmuch():
441+
pass
442+
443+
async def scenario():
444+
gc.collect()
445+
Reenter()
446+
old_threshold = gc.get_threshold()
447+
gc.set_threshold(1)
448+
try:
449+
scratch = [[0], [0], [0]] # schedule a collection
450+
observed['outer'] = task.get_name()
451+
finally:
452+
gc.set_threshold(*old_threshold)
453+
del scratch
454+
455+
# Names are only freeable once the counter is past the small ints.
456+
for _ in range(1100):
457+
loop.run_until_complete(self.new_task(loop, notmuch(), name=None))
458+
task = self.new_task(loop, scenario(), name=None)
459+
loop.run_until_complete(task)
460+
461+
self.assertRegex(observed['outer'], r'^Task-\d+$')
462+
self.assertEqual(observed['outer'], observed['inner'])
463+
self.assertEqual(task.get_name(), observed['outer'])
464+
419465
def test_task_repr_name_not_str(self):
420466
async def notmuch():
421467
return 123

‎Lib/test/test_io/test_memoryio.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -984,6 +984,21 @@ def __index__(self):
984984
finally:
985985
size.result.release()
986986

987+
# Sizes above 2**31 - 1 must work where C long is 32-bit.
988+
if sys.maxsize > 2**31:
989+
stream = self.ioclass(b'abc')
990+
self.assertEqual(stream.truncate(2**31), 2**31)
991+
self.assertEqual(stream.getvalue(), b'abc')
992+
993+
def test_write_overflow(self):
994+
# gh-160781: overallocation at a position near sys.maxsize
995+
# must raise OverflowError, not SystemError.
996+
memio = self.ioclass()
997+
memio.seek(sys.maxsize - 1)
998+
self.assertRaises(OverflowError, memio.write, b"x")
999+
self.assertEqual(memio.getvalue(), b"")
1000+
self.assertEqual(memio.tell(), sys.maxsize - 1)
1001+
9871002
def test_readinto_overlapping_buffer(self):
9881003
with self.ioclass(b'abcd') as stream:
9891004
with stream.getbuffer()[1:] as target:
@@ -1150,6 +1165,12 @@ def __index__(self):
11501165
with self.assertRaises(ValueError):
11511166
stream.truncate(Size())
11521167

1168+
# Sizes above 2**31 - 1 must work where C long is 32-bit.
1169+
if sys.maxsize > 2**31:
1170+
stream = self.ioclass("abc")
1171+
self.assertEqual(stream.truncate(2**31), 2**31)
1172+
self.assertEqual(stream.getvalue(), "abc")
1173+
11531174
# XXX: For the Python version of io.StringIO, this is highly
11541175
# dependent on the encoding used for the underlying buffer.
11551176
def test_widechar(self):

0 commit comments

Comments
 (0)