From f286da096161ae9227959c40f88ceef64e80e7d8 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Mon, 19 Oct 2026 22:34:55 -0400 Subject: [PATCH 1/6] fix(corekit): copy SeekableReader's octets and feed MultiDict a dict, and gate GraalPy's limits (#1963) - SeekableReader copies buffered octets out of its view: GraalPy's BytesIO read/read1/readline from an exported buffer's start can return bytes sharing its memory, which the next buffer slide overwrote - MultiDict hands dict.__init__ and dict.update a dict, not pairs: GraalPy stores pairs through the overridden __setitem__, nesting each value list - new test_graalpy_semantics_1963_unit reproduces both on CPython with stand-ins for GraalPy's behaviour - gate in the tests: pickle protocols 0/1 (typing.Generic is not a heap type), the >sys.maxsize PCAP claim, resource.setrlimit and _abc_impl; bound the huge BytesField length by sys.maxsize Refs #1963 --- pcapkit/corekit/io.py | 32 ++- pcapkit/corekit/multidict.py | 9 +- tests/corekit/test_fields_field.py | 5 +- .../test_graalpy_semantics_1963_unit.py | 189 ++++++++++++++++++ .../test_sentinel_singleton_1842_unit.py | 8 +- tests/dumpkit/test_fallback_cycle_unit.py | 5 +- ...t_pcap_trace_truncated_record_1541_unit.py | 4 + tests/foundation/test_extraction_no_eof.py | 8 +- .../test_mh_timestamp_info_1886_unit.py | 8 +- .../test_frame_incl_len_overrun_1541_unit.py | 4 + tests/protocols/misc/test_pcapng_unit.py | 2 + .../test_schema_metaclass_abc_cache_unit.py | 9 + 12 files changed, 273 insertions(+), 10 deletions(-) create mode 100644 tests/corekit/test_graalpy_semantics_1963_unit.py diff --git a/pcapkit/corekit/io.py b/pcapkit/corekit/io.py index ff49c090de..05df937ff8 100644 --- a/pcapkit/corekit/io.py +++ b/pcapkit/corekit/io.py @@ -286,6 +286,32 @@ def _seek_buffer(self) -> 'int': self._buffer.seek(buf_off, io.SEEK_SET) return self._buffer_cur - buf_off + def _take_buffer(self, size: 'int', /, *, line: 'bool' = False) -> 'bytes': + r"""Read up to ``size`` octets from the buffer's cursor, as a copy, and move past them. + + Args: + size: Octets to read, no more than :meth:`_seek_buffer` says the buffer can serve. + line: Stop after the first ``b'\n'``, as :meth:`io.BytesIO.readline` does. + + Returns: + The octets read. + + Note: + The octets are copied out of :attr:`_buffer_view`, not read with + :meth:`io.BytesIO.read`, ``read1`` or ``readline``. On GraalPy those can hand + back :obj:`bytes` that share the memory of a :class:`io.BytesIO` whose buffer + is exported, as this one's is, so the next :meth:`_write_buffer`, which writes + through the view, changed octets that had already been returned + (:issue:`1963`). A slice of the view, copied, is a copy on every interpreter. + + """ + buf_off = self._buffer.tell() + buf = self._buffer_view[buf_off:buf_off + size].tobytes() + if line and (eol := buf.find(b'\n')) >= 0: + buf = buf[:eol + 1] + self._buffer.seek(buf_off + len(buf), io.SEEK_SET) + return buf + def close(self) -> 'None': """Flush and close this stream. This method has no effect if the file is already closed. Once the file is closed, any operation on the file (e.g. reading or writing) will raise @@ -369,7 +395,7 @@ def readline(self, size: 'int | None' = -1, /) -> 'bytes': buf = b'' else: buf_rem = self._seek_buffer() - buf = self._buffer.readline(buf_rem if size < 0 else min(size, buf_rem)) + buf = self._take_buffer(buf_rem if size < 0 else min(size, buf_rem), line=True) # NOTE: an unbounded ``readline`` has to keep going until the line ends, and # capping it at the buffer's content -- which the line need not end inside -- @@ -651,7 +677,7 @@ def read(self, size: 'int | None' = -1, /) -> 'bytes': buf = b'' else: buf_rem = self._seek_buffer() - buf = self._buffer.read(buf_rem if size < 0 else min(size, buf_rem)) + buf = self._take_buffer(buf_rem if size < 0 else min(size, buf_rem)) size_rem = -1 if size < 0 or (size_rem := size - len(buf)) > 0: @@ -685,7 +711,7 @@ def read1(self, size: 'int | None' = -1, /) -> 'bytes': # NOTE: no continuation from the stream when the buffer answered, since # :meth:`read1` is specified to return only buffered octets if any are. buf_rem = self._seek_buffer() - buf = self._buffer.read1(buf_rem if size < 0 else min(size, buf_rem)) + buf = self._take_buffer(buf_rem if size < 0 else min(size, buf_rem)) if not buf: # pragma: no cover size_rem = -1 diff --git a/pcapkit/corekit/multidict.py b/pcapkit/corekit/multidict.py index bcf3742839..aeebe731b4 100644 --- a/pcapkit/corekit/multidict.py +++ b/pcapkit/corekit/multidict.py @@ -136,7 +136,10 @@ class MultiDict(dict, Generic[_KT, _VT]): def __init__(self, mapping: 'Optional[dict[_KT, _VT] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long if isinstance(mapping, MultiDict): - dict.__init__(self, ((k, v[:]) for k, v in mapping.lists())) + # NOTE: a dict, as the other branches pass, not werkzeug's pairs. GraalPy's + # ``dict.__init__`` stores pairs through the overridden ``__setitem__``, + # which wraps each list in another (:issue:`1963`); a dict it copies as is. + dict.__init__(self, {k: v[:] for k, v in mapping.lists()}) elif isinstance(mapping, dict): tmp = {} # type: dict[_KT, list[_VT]] for key, value in mapping.items(): @@ -159,7 +162,9 @@ def __getstate__(self) -> 'dict[_KT, list[_VT]]': def __setstate__(self, value: 'Iterable[tuple[_KT, list[_VT]]]') -> 'None': dict.clear(self) - dict.update(self, value) # type: ignore[arg-type] + # NOTE: made a dict first, as in :meth:`__init__`: GraalPy's ``dict.update`` + # stores pairs through the overridden ``__setitem__`` (:issue:`1963`). + dict.update(self, dict(value)) # type: ignore[arg-type] def __iter__(self) -> 'Iterator[_KT]': return dict.__iter__(self) diff --git a/tests/corekit/test_fields_field.py b/tests/corekit/test_fields_field.py index d79722f0a0..4258370c92 100644 --- a/tests/corekit/test_fields_field.py +++ b/tests/corekit/test_fields_field.py @@ -1,6 +1,7 @@ from __future__ import annotations import struct +import sys import threading import unittest @@ -173,7 +174,9 @@ def test_wire_declared_length_far_larger_than_the_buffer_is_rejected_without_all in case a future regression reintroduces the allocation instead of just moving the threshold. """ - huge_declared_length = 2 ** 34 # 16 GiB; must never actually be allocated. + # 16 GiB; must never actually be allocated. ``struct`` refuses a count + # past ``sys.maxsize``, which is 2 GiB less one on GraalPy (#1963). + huge_declared_length = min(2 ** 34, sys.maxsize) field = self.BytesField(length=huge_declared_length) with time_limit(): diff --git a/tests/corekit/test_graalpy_semantics_1963_unit.py b/tests/corekit/test_graalpy_semantics_1963_unit.py new file mode 100644 index 0000000000..599028580f --- /dev/null +++ b/tests/corekit/test_graalpy_semantics_1963_unit.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +"""Two GraalPy behaviours :mod:`pcapkit.corekit` no longer depends on (#1963). + +Each was measured on GraalPy 25.4 and does not happen on CPython, so each class +here stands in for GraalPy's behaviour and checks the module does not lean on +CPython's: + +* :class:`io.BytesIO` ``read``, ``read1`` and ``readline`` from the start of an + exported buffer can hand back :obj:`bytes` sharing its memory, so + :class:`~pcapkit.corekit.io.SeekableReader` returned octets that its next write + to the buffer then changed. +* ``dict.__init__`` and ``dict.update`` given ``(key, value)`` pairs store them + through a subclass's ``__setitem__``, so ``MultiDict(MultiDict)``, and a + ``MultiDict`` restored from pairs of its state, wrapped every value list in another. + +On GraalPy itself, ``test_end_of_stream_reads_ahead_only_as_far_as_asked`` in +``test_io_seekable_reader_short_reads_1586_unit.py`` and the ``MultiDict`` copies +in ``test_multidict.py`` fail without the fixes. + +""" + +import builtins +import io +import unittest +from unittest import mock + +from tests._support import reimport_once_per_class + +#: Octets the stream holds: more than a buffer's worth. +DATA = bytes(range(64)) +#: A line longer than GraalPy's shared-read threshold, then more data. +LINES = b'x' * 12 + b'\n' + bytes(range(32)) + + +class AliasingBytesIO(io.BytesIO): + """A buffer whose reads return a view of its own memory, as GraalPy's can. + + On GraalPy, once :meth:`getbuffer` has exported the buffer, a read of more than + eight octets from its start returns :obj:`bytes` that a later write through the + export changes. A :class:`memoryview` of the same octets does the same here. + + """ + + def _view(self, size: 'int | None') -> 'memoryview': + start = self.tell() + view = self.getbuffer() + end = len(view) if size is None or size < 0 else min(start + size, len(view)) + self.seek(end) + return view[start:end] + + def read(self, size: 'int | None' = -1, /) -> 'memoryview': # type: ignore[override] + return self._view(size) + + def read1(self, size: 'int | None' = -1, /) -> 'memoryview': # type: ignore[override] + return self._view(size) + + def readline(self, size: 'int | None' = -1, /) -> 'memoryview': # type: ignore[override] + start = self.tell() + view = self._view(size) + eol = view.tobytes().find(b'\n') + if eol < 0: + return view + self.seek(start + eol + 1) + return view[:eol + 1] + + +class SeekableReaderCopiesItsBufferTests(unittest.TestCase): + """Octets the reader returned stay as they were when it slides its buffer.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def reader(self, data: 'bytes') -> 'object': + from pcapkit.corekit.io import SeekableReader # pylint: disable=import-outside-toplevel + + reader = SeekableReader(io.BytesIO(data), buffer_size=16, stream_closing=False) + # pylint: disable=protected-access + reader._buffer_view.release() + reader._buffer = AliasingBytesIO(bytearray(reader._buffer_size)) + reader._buffer_view = reader._buffer.getbuffer() + self.addCleanup(reader.close) + return reader + + def check(self, method: 'str', data: 'bytes', size: 'int') -> None: + reader = self.reader(data) + self.assertEqual(reader.read(16), data[:16]) + self.assertEqual(reader.seek(0), 0) + + got = getattr(reader, method)(size) + self.assertEqual(got, data[:len(got)]) + self.assertGreater(len(got), 8) + + # the next read from the stream slides the buffer over every octet in it + self.assertEqual(reader.seek(16), 16) + self.assertEqual(reader.read(32), data[16:48]) + self.assertEqual(got, data[:len(got)]) + self.assertIsInstance(got, bytes) + + def test_read(self) -> None: + self.check('read', DATA, 10) + + def test_read1(self) -> None: + self.check('read1', DATA, 10) + + def test_readline(self) -> None: + self.check('readline', LINES, -1) + + +class _PairsThroughSetitemMeta(type): + """Lets :class:`PairsThroughSetitem` stand in for :class:`dict`. + + Calling it builds a plain :class:`dict`, ``isinstance`` asks :class:`dict`, and + every attribute it does not define is :class:`dict`'s. + + """ + + def __call__(cls, *args: 'object', **kwargs: 'object') -> 'dict': + return builtins.dict(*args, **kwargs) + + def __instancecheck__(cls, instance: 'object') -> bool: + return isinstance(instance, builtins.dict) + + def __getattr__(cls, name: 'str') -> 'object': + return getattr(builtins.dict, name) + + +def _store(target: 'dict', args: 'tuple', kwargs: 'dict', method: 'str') -> None: + """Pairs through ``target.__setitem__``, anything else as :class:`dict` stores it.""" + if args and not isinstance(args[0], builtins.dict): + getattr(builtins.dict, method)(target, **kwargs) + for key, value in args[0]: + target[key] = value + return + getattr(builtins.dict, method)(target, *args, **kwargs) + + +class PairsThroughSetitem(metaclass=_PairsThroughSetitemMeta): + """:class:`dict`, but with GraalPy's ``__init__`` and ``update``: pairs go through ``__setitem__``.""" + + def __init__(self, *args: 'object', **kwargs: 'object') -> None: # pylint: disable=super-init-not-called + _store(self, args, kwargs, '__init__') # type: ignore[arg-type] + + def update(self, *args: 'object', **kwargs: 'object') -> None: + _store(self, args, kwargs, 'update') # type: ignore[arg-type] + + +class MultiDictCopyTests(unittest.TestCase): + """``MultiDict(MultiDict)`` keeps each key's values as they were.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_the_stand_in_wraps_pairs_as_graalpy_does(self) -> None: + from pcapkit.corekit.multidict import MultiDict # pylint: disable=import-outside-toplevel + + for method in (PairsThroughSetitem.__init__, PairsThroughSetitem.update): + with self.subTest(method=method.__name__): + data = MultiDict() + method(data, [('a', [5, 6])]) + self.assertEqual(dict.__getitem__(data, 'a'), [[5, 6]]) + method(data, {'b': [7]}) + self.assertEqual(dict.__getitem__(data, 'b'), [7]) + + def test_a_copy_keeps_the_value_lists(self) -> None: + from pcapkit.corekit import multidict # pylint: disable=import-outside-toplevel + + for cls in (multidict.MultiDict, multidict.OrderedMultiDict): + with self.subTest(cls=cls.__name__): + data = multidict.MultiDict([('a', 5), ('a', 6), ('b', 3)]) + with mock.patch.object(multidict, 'dict', PairsThroughSetitem, create=True): + copied = cls(data) + self.assertEqual(copied.getlist('a'), [5, 6]) + self.assertEqual(copied.getlist('b'), [3]) + + def test_state_restored_from_pairs_keeps_the_value_lists(self) -> None: + from pcapkit.corekit import multidict # pylint: disable=import-outside-toplevel + + state = multidict.MultiDict([('a', 5), ('a', 6), ('b', 3)]).__getstate__() + for given in (state, state.items()): + with self.subTest(state=type(given).__name__): + restored = multidict.MultiDict() + with mock.patch.object(multidict, 'dict', PairsThroughSetitem, create=True): + restored.__setstate__(given) + self.assertEqual(restored.getlist('a'), [5, 6]) + self.assertEqual(restored.getlist('b'), [3]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/corekit/test_sentinel_singleton_1842_unit.py b/tests/corekit/test_sentinel_singleton_1842_unit.py index 2c8c81ca18..fd7ceecc32 100644 --- a/tests/corekit/test_sentinel_singleton_1842_unit.py +++ b/tests/corekit/test_sentinel_singleton_1842_unit.py @@ -33,6 +33,7 @@ import importlib import inspect import pickle +import sys import unittest from typing import TYPE_CHECKING @@ -57,6 +58,11 @@ #: Python's own singletons, which a sentinel is to behave like. BUILTINS = (None, Ellipsis, NotImplemented) +#: The lowest :mod:`pickle` protocol a :class:`typing.Generic` subclass round-trips at. +#: GraalPy's :class:`typing.Generic` is not a heap type, so ``copyreg._reduce_ex``, +#: which protocols 0 and 1 use, stops at it and fails on ``Generic(obj)`` (#1963). +LOWEST_PROTOCOL = 2 if sys.implementation.name == 'graalpy' else 0 + #: Pickles written at ``75a9f237b``, keyed by ``(object name, protocol)``. ``NULL`` #: reduced to the module-level ``_get_null``; the others took the default reduce, #: which is ``copyreg._reconstructor`` below protocol 2 and ``NEWOBJ`` from 2 on. @@ -440,7 +446,7 @@ def test_a_parsed_mpl_seed_keeps_its_identity(self) -> 'None': bad = [] if copy.deepcopy(schema).options[0].seed is not no_value: bad.append('deepcopy') - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + for protocol in range(LOWEST_PROTOCOL, pickle.HIGHEST_PROTOCOL + 1): if pickle.loads(pickle.dumps(schema, protocol=protocol)).options[0].seed is not no_value: bad.append(protocol) self.assertEqual(bad, []) diff --git a/tests/dumpkit/test_fallback_cycle_unit.py b/tests/dumpkit/test_fallback_cycle_unit.py index 13d4cea895..3ec2a7790d 100644 --- a/tests/dumpkit/test_fallback_cycle_unit.py +++ b/tests/dumpkit/test_fallback_cycle_unit.py @@ -19,6 +19,7 @@ import json import os import plistlib +import sys import tempfile import unittest from unittest import mock @@ -53,7 +54,9 @@ def setUp(self) -> None: reimport_once_per_class(self) self.tmpdir = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with self.addCleanup(self.tmpdir.cleanup) - if resource is not None: + # NOTE: GraalPy's ``resource`` has no ``setrlimit`` (#1963), so there, as + # off POSIX, the file is left to the leg's own cap. + if resource is not None and sys.implementation.name != 'graalpy': # NOTE: without the fix, the class case writes without bound, so # cap the file size rather than let it fill the disk. limits = resource.getrlimit(resource.RLIMIT_FSIZE) diff --git a/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py b/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py index 7ee7930e38..8caf566405 100644 --- a/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py +++ b/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py @@ -21,6 +21,7 @@ import io import os import struct +import sys import tempfile import unittest import warnings @@ -110,6 +111,9 @@ def test_a_pcap_record_cut_at_eof_is_traced_as_the_octets_it_has(self) -> None: """#1541: a record claiming more than the file holds writes what it holds.""" for claim in CLAIMS: with self.subTest(claim=claim): + if 16 + claim > sys.maxsize: + self.skipTest('the frame reader asks the file for the 16-octet header and the claim at once, ' + 'past sys.maxsize on GraalPy (#1963)') self.check(self.trace(_cut_pcap(claim)), _frames()[-1]) def test_a_pcapng_captured_length_past_its_block_is_traced_as_the_octets_it_has(self) -> None: diff --git a/tests/foundation/test_extraction_no_eof.py b/tests/foundation/test_extraction_no_eof.py index 6b66bbd08f..9df6a21600 100644 --- a/tests/foundation/test_extraction_no_eof.py +++ b/tests/foundation/test_extraction_no_eof.py @@ -126,11 +126,17 @@ #: "the extraction did not finish" from "the child failed for some other reason". SENTINEL = 'NO-EOF-RETURNED' +#: How a child caps its address space. GraalPy's ``resource`` has no ``setrlimit`` +#: (#1963); a child there is held by the heap cap the leg gives every GraalPy +#: process, in ``GRAAL_PYTHON_VM_ARGS``, which it inherits. +CHILD_CAP = ('' if sys.implementation.name == 'graalpy' + else 'resource.setrlimit(resource.RLIMIT_AS, ({cap}, {cap}))') + #: Preamble for every child: cap the address space, put the tree under test first #: on ``sys.path``, and *prove* that is the tree which got imported. CHILD_PREAMBLE = """ import resource, sys, warnings -resource.setrlimit(resource.RLIMIT_AS, ({cap}, {cap})) +""" + CHILD_CAP + """ sys.path.insert(0, {root!r}) warnings.simplefilter('ignore') import pcapkit diff --git a/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py b/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py index b000b5541b..404ac34ab7 100644 --- a/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py +++ b/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py @@ -24,6 +24,7 @@ import pickle import plistlib import struct +import sys import tempfile import unittest import warnings @@ -33,6 +34,11 @@ RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) +#: The lowest :mod:`pickle` protocol a :class:`typing.Generic` subclass round-trips at. +#: GraalPy's :class:`typing.Generic` is not a heap type, so ``copyreg._reduce_ex``, +#: which protocols 0 and 1 use, stops at it and fails on ``Generic(obj)`` (#1963). +LOWEST_PROTOCOL = 2 if sys.implementation.name == 'graalpy' else 0 + #: NTP seconds and fraction of the Mobility Message Replay Protection option: #: the UNIX epoch counted from 1900, plus a quarter of a second. NTP = (0x83aa7e80, 0x40000000) @@ -163,7 +169,7 @@ def test_pickling_round_trips(self) -> None: from pcapkit.corekit.infoclass import Info _, mesg_id, timestamp = self.parse() - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + for protocol in range(LOWEST_PROTOCOL, pickle.HIGHEST_PROTOCOL + 1): for value in (mesg_id.ntp_timestamp, timestamp.pmip_timestamp, mesg_id, timestamp): with self.subTest(protocol=protocol, type=type(value).__name__): restored = pickle.loads(pickle.dumps(value, protocol)) diff --git a/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py b/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py index 4248096bf0..5e575a6a80 100644 --- a/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py +++ b/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py @@ -15,6 +15,7 @@ import io import struct +import sys import unittest import warnings @@ -54,6 +55,9 @@ def test_overrun_warns_and_is_kept_as_captured(self) -> None: header = Header(GLOBAL).info for claim in (61, 1000, 0x7FFF_FFFF): with self.subTest(claim=claim): + if 16 + claim > sys.maxsize: + self.skipTest('the frame reader asks the file for the 16-octet header and the claim at once, ' + 'past sys.maxsize on GraalPy (#1963)') octets = _record(claim) frame, messages = self.parse(octets) self.assertIn(f'PCAP: [Frame 4] captured length {claim} runs past the data, ' diff --git a/tests/protocols/misc/test_pcapng_unit.py b/tests/protocols/misc/test_pcapng_unit.py index a5a5243026..a2fa28f7bc 100644 --- a/tests/protocols/misc/test_pcapng_unit.py +++ b/tests/protocols/misc/test_pcapng_unit.py @@ -4756,6 +4756,8 @@ def test_the_captured_len_vector_parses_under_a_memory_cap(self) -> None: """ if importlib.util.find_spec('resource') is None: # pragma: no cover self.skipTest('the resource module is unavailable on this platform') + if sys.implementation.name == 'graalpy': # pragma: no cover + self.skipTest('GraalPy has no resource.setrlimit to cap the subprocess with (#1963)') source = textwrap.dedent(''' import io, json, resource, struct, sys, warnings diff --git a/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py b/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py index 9f20c72c2a..bcb8ac5d6e 100644 --- a/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py +++ b/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import sys import unittest from collections import ChainMap from collections.abc import Mapping @@ -10,6 +11,11 @@ RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) +#: GraalPy's :mod:`abc` is ``_py_abc``, which keeps its registry and caches as separate +#: attributes, not in the C accelerator's ``_abc_impl`` these tests inspect (#1963). +NO_ABC_IMPL = sys.implementation.name == 'graalpy' +NO_ABC_IMPL_REASON = "GraalPy's abc is _py_abc, which has no _abc_impl (#1963)" + @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') class SchemaMetaAbcCacheTests(unittest.TestCase): @@ -109,6 +115,7 @@ def test_asking_schema_first_does_not_break_mapping_for_a_plain_dict(self) -> No # THEN ask Mapping. A plain dict must still be reported as one. self.assertTrue(isinstance({}, Mapping)) + @unittest.skipIf(NO_ABC_IMPL, NO_ABC_IMPL_REASON) def test_schema_subclasses_have_their_own_abc_impl(self) -> None: """Every schema class gets its own ABCMeta registry/cache triple. @@ -177,6 +184,7 @@ class _SharedChild(_Shared): del _SharedChild._abc_impl return _Shared, _SharedChild + @unittest.skipIf(NO_ABC_IMPL, NO_ABC_IMPL_REASON) def test_probing_the_same_exact_type_reproduces_439s_shape(self) -> None: """The general mechanism, generalised beyond :class:`dict` and ``Schema``. @@ -197,6 +205,7 @@ def test_probing_the_same_exact_type_reproduces_439s_shape(self) -> None: self.assertTrue(isinstance(ChainMap(), Mapping)) self.assertTrue(isinstance(ChainMap(), _SharedChild)) + @unittest.skipIf(NO_ABC_IMPL, NO_ABC_IMPL_REASON) def test_probing_a_different_exact_type_gives_a_false_negative(self) -> None: """A pitfall in reproducing #439: the cache is keyed by *exact* type. From 9dced17303764b0a1f36f2284838ff92eb3cbeec Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Mon, 19 Oct 2026 22:50:51 -0400 Subject: [PATCH 2/6] test(corekit): keep only the cases that fail without the GraalPy fixes (#1963) - drop the OrderedMultiDict copy and the plain-dict state cases, which passed without the fixes - say why the stand-in check stays: it guards the stand-in, not the fix - drop the count from the module docstring - correction to the first commit's review: werkzeug 2.3.8's MultiDict.__init__ and __setstate__ match pcapkit's apart from the annotations and the l -> v rename Refs #1963 --- .../test_graalpy_semantics_1963_unit.py | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/corekit/test_graalpy_semantics_1963_unit.py b/tests/corekit/test_graalpy_semantics_1963_unit.py index 599028580f..0e329865eb 100644 --- a/tests/corekit/test_graalpy_semantics_1963_unit.py +++ b/tests/corekit/test_graalpy_semantics_1963_unit.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Two GraalPy behaviours :mod:`pcapkit.corekit` no longer depends on (#1963). +"""GraalPy behaviours :mod:`pcapkit.corekit` no longer depends on (#1963). Each was measured on GraalPy 25.4 and does not happen on CPython, so each class here stands in for GraalPy's behaviour and checks the module does not lean on @@ -151,6 +151,13 @@ def setUp(self) -> None: reimport_once_per_class(self) def test_the_stand_in_wraps_pairs_as_graalpy_does(self) -> None: + """A guard on the stand-in, not on the fix: it passes with or without it. + + The tests below fail without the fix only because the stand-in stores + pairs through ``__setitem__``, as GraalPy does, and a plain dict as is. + Without this, a stand-in that lost either would let them pass vacuously. + + """ from pcapkit.corekit.multidict import MultiDict # pylint: disable=import-outside-toplevel for method in (PairsThroughSetitem.__init__, PairsThroughSetitem.update): @@ -164,25 +171,21 @@ def test_the_stand_in_wraps_pairs_as_graalpy_does(self) -> None: def test_a_copy_keeps_the_value_lists(self) -> None: from pcapkit.corekit import multidict # pylint: disable=import-outside-toplevel - for cls in (multidict.MultiDict, multidict.OrderedMultiDict): - with self.subTest(cls=cls.__name__): - data = multidict.MultiDict([('a', 5), ('a', 6), ('b', 3)]) - with mock.patch.object(multidict, 'dict', PairsThroughSetitem, create=True): - copied = cls(data) - self.assertEqual(copied.getlist('a'), [5, 6]) - self.assertEqual(copied.getlist('b'), [3]) + data = multidict.MultiDict([('a', 5), ('a', 6), ('b', 3)]) + with mock.patch.object(multidict, 'dict', PairsThroughSetitem, create=True): + copied = multidict.MultiDict(data) + self.assertEqual(copied.getlist('a'), [5, 6]) + self.assertEqual(copied.getlist('b'), [3]) def test_state_restored_from_pairs_keeps_the_value_lists(self) -> None: from pcapkit.corekit import multidict # pylint: disable=import-outside-toplevel state = multidict.MultiDict([('a', 5), ('a', 6), ('b', 3)]).__getstate__() - for given in (state, state.items()): - with self.subTest(state=type(given).__name__): - restored = multidict.MultiDict() - with mock.patch.object(multidict, 'dict', PairsThroughSetitem, create=True): - restored.__setstate__(given) - self.assertEqual(restored.getlist('a'), [5, 6]) - self.assertEqual(restored.getlist('b'), [3]) + restored = multidict.MultiDict() + with mock.patch.object(multidict, 'dict', PairsThroughSetitem, create=True): + restored.__setstate__(state.items()) + self.assertEqual(restored.getlist('a'), [5, 6]) + self.assertEqual(restored.getlist('b'), [3]) if __name__ == '__main__': From a81cc522515aaba84ce282d521eb92babeff2862 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 00:22:30 -0400 Subject: [PATCH 3/6] fix(corekit): upgrade the MultiDict port to werkzeug 3.1.9 (#1963) - port MultiDict, OrderedMultiDict and iter_multi_items from werkzeug 3.1.9 rather than 2.3.8: dict reached through super(), any Mapping and set values accepted, | and |= added, setdefault/setlistdefault simplified - keep the local adaptations: dict base without TypeConversionDict, MissingKeyError/UnsupportedCall, get() and getlist() without type=, OrderedMultiDict.deepcopy order fix, and no DeprecationWarning on OrderedMultiDict - werkzeug 3.1.9 still feeds pairs to dict.__init__ and update, which GraalPy routes through __setitem__; keep the two dict-building lines as the only behavioural divergence, noted in place - the GraalPy stand-in now sits in the MRO behind MultiDict, since super() no longer reads a module-global dict - test the new Mapping/set handling and the | operators tests/corekit and every test module importing the multidict pass on CPython; the #1963 and multidict tests pass on GraalPy 25.4.4. Refs #1963 --- pcapkit/corekit/multidict.py | 162 +++++++++++------- .../test_graalpy_semantics_1963_unit.py | 81 ++++----- tests/corekit/test_multidict.py | 29 ++++ 3 files changed, 166 insertions(+), 106 deletions(-) diff --git a/pcapkit/corekit/multidict.py b/pcapkit/corekit/multidict.py index aeebe731b4..b508f1ac32 100644 --- a/pcapkit/corekit/multidict.py +++ b/pcapkit/corekit/multidict.py @@ -6,10 +6,11 @@ :mod:`pcapkit.corekit.multidict` contains multi-mapping dictionary classes, which are used to store multiple mappings of the same key. The implementation -is inspired and based on the `Werkzeug`_ project. +is ported from the `Werkzeug`_ project, as of its release 3.1.9. """ +import collections.abc import copy from typing import TYPE_CHECKING, Generic, TypeVar, cast, overload @@ -18,7 +19,7 @@ if TYPE_CHECKING: from typing import Any, Iterable, Iterator, Mapping, NoReturn, Optional, SupportsIndex - from typing_extensions import Literal + from typing_extensions import Literal, Self __all__ = ['MultiDict', 'OrderedMultiDict'] @@ -88,15 +89,21 @@ def __reduce__(self) -> 'str': ############################################################################### -def iter_multi_items(mapping: 'Mapping[_KT, _VT | Iterable[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'Iterator[tuple[_KT, _VT]]': # pylint: disable=line-too-long +def iter_multi_items(mapping: 'MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'Iterator[tuple[_KT, _VT]]': # pylint: disable=line-too-long """Iterates over the items of a mapping yielding keys and values without dropping any from more complex structures. + + .. versionchanged:: 1.5.0 + + Takes any :class:`~collections.abc.Mapping`, not only a :obj:`dict`, and + yields each value of a :obj:`set` value, as Werkzeug 3.1 does. + """ if isinstance(mapping, MultiDict): yield from mapping.items(multi=True) - elif isinstance(mapping, dict): + elif isinstance(mapping, collections.abc.Mapping): for key, value in mapping.items(): - if isinstance(value, (tuple, list)): + if isinstance(value, (list, tuple, set)): for v in value: yield key, v else: @@ -122,52 +129,66 @@ class MultiDict(dict, Generic[_KT, _VT]): Args: mapping: The initial value for the :class:`MultiDict`. Either a - regular :obj:`dict`, an iterable of ``(key, value)`` tuples, or - :obj:`None`. + :class:`MultiDict`, a regular :class:`~collections.abc.Mapping`, an + iterable of ``(key, value)`` tuples, or :obj:`None`. It behaves like a normal :obj:`dict` thus all :obj:`dict` functions will only return the first value when multiple values for one key are found. See Also: - The class is inspired from and based on the `Werkzeug`_ project (cf. + The class is ported from the `Werkzeug`_ project, release 3.1.9 (cf. ``werkzeug.datastructures.MultiDict``). + .. versionchanged:: 1.5.0 + + Implements the ``|`` and ``|=`` operators, takes any + :class:`~collections.abc.Mapping` and :obj:`set` values, and calls + :obj:`dict` through :func:`super`, as Werkzeug 3.1 does. + """ - def __init__(self, mapping: 'Optional[dict[_KT, _VT] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long - if isinstance(mapping, MultiDict): - # NOTE: a dict, as the other branches pass, not werkzeug's pairs. GraalPy's - # ``dict.__init__`` stores pairs through the overridden ``__setitem__``, - # which wraps each list in another (:issue:`1963`); a dict it copies as is. - dict.__init__(self, {k: v[:] for k, v in mapping.lists()}) - elif isinstance(mapping, dict): + def __init__(self, mapping: 'Optional[MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long + if mapping is None: + super().__init__() + elif isinstance(mapping, MultiDict): + # NOTE: a dict, as the other branches pass, where Werkzeug passes pairs. + # GraalPy's ``dict.__init__`` stores pairs through the overridden + # ``__setitem__``, which wraps each list in another (:issue:`1963`); a + # dict it copies as is. CPython stores either as is. + super().__init__({k: vs[:] for k, vs in mapping.lists()}) + elif isinstance(mapping, collections.abc.Mapping): tmp = {} # type: dict[_KT, list[_VT]] for key, value in mapping.items(): - if isinstance(value, (tuple, list)): - if len(value) == 0: - continue + if isinstance(value, (list, tuple, set)): value = list(value) + + if not value: + continue else: value = [value] tmp[key] = value - dict.__init__(self, tmp) + super().__init__(tmp) else: tmp = {} - for key, value in mapping or (): + for key, value in mapping: # type: ignore[misc] tmp.setdefault(key, []).append(value) - dict.__init__(self, tmp) + super().__init__(tmp) def __getstate__(self) -> 'dict[_KT, list[_VT]]': return dict(self.lists()) - def __setstate__(self, value: 'Iterable[tuple[_KT, list[_VT]]]') -> 'None': - dict.clear(self) - # NOTE: made a dict first, as in :meth:`__init__`: GraalPy's ``dict.update`` - # stores pairs through the overridden ``__setitem__`` (:issue:`1963`). - dict.update(self, dict(value)) # type: ignore[arg-type] + def __setstate__(self, value: 'Mapping[_KT, list[_VT]] | Iterable[tuple[_KT, list[_VT]]]') -> 'None': + super().clear() + # NOTE: made a dict first, where Werkzeug passes ``value`` on as is: given + # pairs, GraalPy's ``dict.update`` stores them through the overridden + # ``__setitem__``, as its ``dict.__init__`` does (:issue:`1963`). + super().update(dict(value)) def __iter__(self) -> 'Iterator[_KT]': - return dict.__iter__(self) + # NOTE: https://github.com/python/cpython/issues/87412 -- without this + # override, ``dict(md)`` takes the value lists directly instead of calling + # :meth:`__getitem__` for the first value of each key. + return super().__iter__() def __getitem__(self, key: '_KT') -> '_VT': """Return the first data value for this key. @@ -180,7 +201,7 @@ def __getitem__(self, key: '_KT') -> '_VT': """ if key in self: - lst = dict.__getitem__(self, key) + lst = super().__getitem__(key) if len(lst) > 0: return lst[0] raise MissingKeyError(key, quiet=True) @@ -193,7 +214,7 @@ def __setitem__(self, key: '_KT', value: '_VT') -> 'None': value: The value to set. """ - dict.__setitem__(self, key, [value]) + super().__setitem__(key, [value]) def add(self, key: '_KT', value: '_VT') -> 'None': """Adds a new value for the key. @@ -203,7 +224,7 @@ def add(self, key: '_KT', value: '_VT') -> 'None': value: The value to add. """ - dict.setdefault(self, key, []).append(value) # type: ignore[arg-type,attr-defined] + super().setdefault(key, []).append(value) # type: ignore[arg-type,attr-defined] @overload def get(self, key: '_KT') -> '_VT | _T': ... @@ -230,7 +251,7 @@ def getlist(self, key: '_KT') -> 'list[_VT]': """ try: - rv = dict.__getitem__(self, key) # type: list[_VT] + rv = super().__getitem__(key) # type: list[_VT] except KeyError: return [] return list(rv) @@ -248,7 +269,7 @@ def setlist(self, key: '_KT', new_list: 'Iterable[_VT]') -> 'None': are removed first. """ - dict.__setitem__(self, key, list(new_list)) + super().__setitem__(key, list(new_list)) def setdefault(self, key: '_KT', default: 'Optional[_VT]' = None) -> '_VT': """If key is in the dictionary, returns its value. @@ -265,9 +286,8 @@ def setdefault(self, key: '_KT', default: 'Optional[_VT]' = None) -> '_VT': """ if key not in self: self[key] = cast('_VT', default) - else: - default = self[key] - return default # type: ignore[return-value] + + return self[key] def setlistdefault(self, key: '_KT', default_list: 'Optional[Iterable[_VT]]' = None) -> 'list[_VT]': """Like :meth:`setdefault` but sets multiple values. @@ -284,11 +304,9 @@ def setlistdefault(self, key: '_KT', default_list: 'Optional[Iterable[_VT]]' = N """ if key not in self: - default_list = list(default_list or ()) - dict.__setitem__(self, key, default_list) - else: - default_list = cast('list[_VT]', dict.__getitem__(self, key)) - return default_list + super().__setitem__(key, list(default_list or ())) + + return cast('list[_VT]', super().__getitem__(key)) def items(self, multi: 'bool' = False) -> 'Iterable[tuple[_KT, _VT]]': # type: ignore[override] """Return an interator of ``(key, value)`` paris. @@ -299,7 +317,7 @@ def items(self, multi: 'bool' = False) -> 'Iterable[tuple[_KT, _VT]]': # type: pairs for the first value of each key. """ - for key, values in dict.items(self): + for key, values in super().items(): if multi: for value in values: yield key, value @@ -309,18 +327,18 @@ def items(self, multi: 'bool' = False) -> 'Iterable[tuple[_KT, _VT]]': # type: def lists(self) -> 'Iterator[tuple[_KT, list[_VT]]]': """Return an iterator of ``(key, values)`` pairs, where ``values`` is the :obj:`list` of all values associated with the key.""" - for key, values in dict.items(self): + for key, values in super().items(): yield key, list(values) def values(self) -> 'Iterator[_VT]': # type: ignore[override] """Returns an iterator of the first value on every key's value list.""" - for values in dict.values(self): + for values in super().values(): yield values[0] def listvalues(self) -> 'Iterator[list[_VT]]': """Return an iterator of all values associated with a key. Zipping :meth:`keys` and this is the same as calling :meth:`lists`.""" - return dict.values(self) # type: ignore[return-value] + return super().values() # type: ignore[return-value] def copy(self) -> 'MultiDict[_KT, _VT]': """Return a shallow copy of this object.""" @@ -328,7 +346,7 @@ def copy(self) -> 'MultiDict[_KT, _VT]': def deepcopy(self, memo: 'Optional[dict]' = None) -> 'MultiDict[_KT, _VT]': """Return a deep copy of this object.""" - return self.__class__(copy.deepcopy(self.to_dict(flat=False), memo=memo)) # type: ignore[arg-type] + return self.__class__(copy.deepcopy(self.to_dict(flat=False), memo=memo)) @overload def to_dict(self, flat: 'Literal[True]' = ...) -> 'dict[_KT, _VT]': ... @@ -352,7 +370,7 @@ def to_dict(self, flat: 'bool' = True) -> 'dict[_KT, _VT] | dict[_KT, list[_VT]] return dict(self.items()) return dict(self.lists()) - def update(self, mapping: 'Mapping[_KT, _VT] | Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] + def update(self, mapping: 'MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] # pylint: disable=line-too-long """:meth:`update` extends rather than replaces existing key lists. If the value :obj:`list` for a key in ``other_dict`` is empty, no new @@ -361,12 +379,27 @@ def update(self, mapping: 'Mapping[_KT, _VT] | Iterable[tuple[_KT, _VT]]') -> 'N Args: mapping: The extending value for the :class:`MultiDict`. Either a - regular :obj:`dict`, an iterable of ``(key, value)`` tuples, or - :obj:`None`. + :class:`MultiDict`, a regular :class:`~collections.abc.Mapping`, + or an iterable of ``(key, value)`` tuples. """ for key, value in iter_multi_items(mapping): - MultiDict.add(self, key, value) + self.add(key, value) + + def __or__(self, other: 'Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]]') -> 'MultiDict[_KT, _VT]': # type: ignore[override] # pylint: disable=line-too-long + if not isinstance(other, collections.abc.Mapping): + return NotImplemented + + rv = self.copy() + rv.update(other) + return rv + + def __ior__(self, other: 'Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'Self': # type: ignore[override,misc] # pylint: disable=line-too-long + if not isinstance(other, (collections.abc.Mapping, collections.abc.Iterable)): + return NotImplemented + + self.update(other) + return self @overload def pop(self, key: '_KT') -> '_VT': ... @@ -386,7 +419,7 @@ def pop(self, key: '_KT', default: '_VT | _T' = _missing) -> '_VT | _T': # type """ try: - lst = dict.pop(self, key) + lst = super().pop(key) if len(lst) == 0: raise MissingKeyError(key) @@ -401,7 +434,7 @@ def pop(self, key: '_KT', default: '_VT | _T' = _missing) -> '_VT | _T': # type def popitem(self) -> 'tuple[_KT, _VT]': """Pop an item from the :obj:`dict`.""" try: - item = dict.popitem(self) + item = super().popitem() if len(item[1]) == 0: # type: ignore[arg-type] raise MissingKeyError(item[0]) @@ -423,12 +456,12 @@ def poplist(self, key: '_KT') -> 'list[_VT]': key: The key to pop. """ - return dict.pop(self, key, []) + return super().pop(key, []) def popitemlist(self) -> 'tuple[_KT, list[_VT]]': """Pop a ``(key, list)`` :obj:`tuple` from the :obj:`dict`.""" try: - return dict.popitem(self) # type: ignore[return-value] + return super().popitem() # type: ignore[return-value] except KeyError as e: raise MissingKeyError(e.args[0]) from None @@ -448,8 +481,8 @@ class OrderedMultiDict(MultiDict[_KT, _VT]): Args: mapping: The initial value for the :class:`MultiDict`. Either a - regular :obj:`dict`, an iterable of ``(key, value)`` tuples, or - :obj:`None`. + :class:`MultiDict`, a regular :class:`~collections.abc.Mapping`, an + iterable of ``(key, value)`` tuples, or :obj:`None`. To convert the ordered multi dict into a :obj:`list` you can us the :meth:`items` method and pass it ``multi=True``. @@ -463,15 +496,19 @@ class OrderedMultiDict(MultiDict[_KT, _VT]): have to use the :meth:`to_dict` method, otherwise the internal bucket objects are exposed. + Werkzeug 3.1 deprecates its ``OrderedMultiDict``, to be removed in 3.2; this + port keeps the class, without the :exc:`DeprecationWarning`. + """ _first_bucket: 'Optional[_omd_bucket[_KT, _VT]]' _last_bucket: 'Optional[_omd_bucket[_KT, _VT]]' - def __init__(self, mapping: 'Optional[dict[_KT, _VT] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long - dict.__init__(self) # pylint: disable=non-parent-init-called - self._first_bucket = self._last_bucket = None + def __init__(self, mapping: 'Optional[MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long + super().__init__() + self._first_bucket = None + self._last_bucket = None if mapping is not None: - OrderedMultiDict.update(self, mapping) + self.update(mapping) def __eq__(self, other: 'Any') -> 'bool': if not isinstance(other, MultiDict): @@ -524,7 +561,8 @@ def __getstate__(self) -> 'list[tuple[_KT, _VT]]': # type: ignore[override] return list(self.items(multi=True)) def __setstate__(self, values: 'Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] - dict.clear(self) + self.clear() + for key, value in values: self.add(key, value) @@ -594,9 +632,9 @@ def setlist(self, key: '_KT', new_list: 'Iterable[_VT]') -> 'None': def setlistdefault(self, key: '_KT', default_list: 'Optional[Iterable[_VT]]' = None) -> 'NoReturn': raise UnsupportedCall('setlistdefault is unsupported for ordered multi dicts') - def update(self, mapping: 'Mapping[_KT, _VT] | Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] + def update(self, mapping: 'MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] # pylint: disable=line-too-long for key, value in iter_multi_items(mapping): - OrderedMultiDict.add(self, key, value) + self.add(key, value) def poplist(self, key: '_KT') -> 'list[_VT]': buckets = dict.pop(self, key, []) # type: list[_omd_bucket[_KT, _VT]] diff --git a/tests/corekit/test_graalpy_semantics_1963_unit.py b/tests/corekit/test_graalpy_semantics_1963_unit.py index 0e329865eb..3488d25770 100644 --- a/tests/corekit/test_graalpy_semantics_1963_unit.py +++ b/tests/corekit/test_graalpy_semantics_1963_unit.py @@ -19,10 +19,8 @@ """ -import builtins import io import unittest -from unittest import mock from tests._support import reimport_once_per_class @@ -106,42 +104,29 @@ def test_readline(self) -> None: self.check('readline', LINES, -1) -class _PairsThroughSetitemMeta(type): - """Lets :class:`PairsThroughSetitem` stand in for :class:`dict`. +class PairsThroughSetitem(dict): + """:class:`dict`, with GraalPy's ``__init__`` and ``update``: pairs go through ``__setitem__``. - Calling it builds a plain :class:`dict`, ``isinstance`` asks :class:`dict`, and - every attribute it does not define is :class:`dict`'s. + :class:`~pcapkit.corekit.multidict.MultiDict` reaches :obj:`dict` through + :func:`super`, as Werkzeug's does, so a class deriving from it and then from + this one puts this in its place, as GraalPy's :obj:`dict` would be. """ - def __call__(cls, *args: 'object', **kwargs: 'object') -> 'dict': - return builtins.dict(*args, **kwargs) - - def __instancecheck__(cls, instance: 'object') -> bool: - return isinstance(instance, builtins.dict) - - def __getattr__(cls, name: 'str') -> 'object': - return getattr(builtins.dict, name) - - -def _store(target: 'dict', args: 'tuple', kwargs: 'dict', method: 'str') -> None: - """Pairs through ``target.__setitem__``, anything else as :class:`dict` stores it.""" - if args and not isinstance(args[0], builtins.dict): - getattr(builtins.dict, method)(target, **kwargs) - for key, value in args[0]: - target[key] = value - return - getattr(builtins.dict, method)(target, *args, **kwargs) - - -class PairsThroughSetitem(metaclass=_PairsThroughSetitemMeta): - """:class:`dict`, but with GraalPy's ``__init__`` and ``update``: pairs go through ``__setitem__``.""" - def __init__(self, *args: 'object', **kwargs: 'object') -> None: # pylint: disable=super-init-not-called - _store(self, args, kwargs, '__init__') # type: ignore[arg-type] + self._store(dict.__init__, args, kwargs) def update(self, *args: 'object', **kwargs: 'object') -> None: - _store(self, args, kwargs, 'update') # type: ignore[arg-type] + self._store(dict.update, args, kwargs) + + def _store(self, method: 'object', args: 'tuple', kwargs: 'dict') -> None: + """Pairs through ``self.__setitem__``, anything else as :class:`dict` stores it.""" + if args and not isinstance(args[0], dict): + method(self, **kwargs) # type: ignore[operator] + for key, value in args[0]: + self[key] = value + return + method(self, *args, **kwargs) # type: ignore[operator] class MultiDictCopyTests(unittest.TestCase): @@ -150,6 +135,15 @@ class MultiDictCopyTests(unittest.TestCase): def setUp(self) -> None: reimport_once_per_class(self) + def multidict(self) -> 'type': + """A :class:`~pcapkit.corekit.multidict.MultiDict` over :class:`PairsThroughSetitem`.""" + from pcapkit.corekit.multidict import MultiDict # pylint: disable=import-outside-toplevel + + class GraalMultiDict(MultiDict, PairsThroughSetitem): # type: ignore[misc] + """A :class:`~pcapkit.corekit.multidict.MultiDict` on GraalPy's :obj:`dict`.""" + + return GraalMultiDict + def test_the_stand_in_wraps_pairs_as_graalpy_does(self) -> None: """A guard on the stand-in, not on the fix: it passes with or without it. @@ -158,32 +152,31 @@ def test_the_stand_in_wraps_pairs_as_graalpy_does(self) -> None: Without this, a stand-in that lost either would let them pass vacuously. """ - from pcapkit.corekit.multidict import MultiDict # pylint: disable=import-outside-toplevel + cls = self.multidict() + # MultiDict's ``super()`` reaches the stand-in, not :obj:`dict` + self.assertEqual([c.__name__ for c in cls.__mro__[1:4]], + ['MultiDict', 'PairsThroughSetitem', 'dict']) for method in (PairsThroughSetitem.__init__, PairsThroughSetitem.update): with self.subTest(method=method.__name__): - data = MultiDict() + data = cls() method(data, [('a', [5, 6])]) self.assertEqual(dict.__getitem__(data, 'a'), [[5, 6]]) method(data, {'b': [7]}) self.assertEqual(dict.__getitem__(data, 'b'), [7]) def test_a_copy_keeps_the_value_lists(self) -> None: - from pcapkit.corekit import multidict # pylint: disable=import-outside-toplevel - - data = multidict.MultiDict([('a', 5), ('a', 6), ('b', 3)]) - with mock.patch.object(multidict, 'dict', PairsThroughSetitem, create=True): - copied = multidict.MultiDict(data) + cls = self.multidict() + data = cls([('a', 5), ('a', 6), ('b', 3)]) + copied = cls(data) self.assertEqual(copied.getlist('a'), [5, 6]) self.assertEqual(copied.getlist('b'), [3]) def test_state_restored_from_pairs_keeps_the_value_lists(self) -> None: - from pcapkit.corekit import multidict # pylint: disable=import-outside-toplevel - - state = multidict.MultiDict([('a', 5), ('a', 6), ('b', 3)]).__getstate__() - restored = multidict.MultiDict() - with mock.patch.object(multidict, 'dict', PairsThroughSetitem, create=True): - restored.__setstate__(state.items()) + cls = self.multidict() + state = cls([('a', 5), ('a', 6), ('b', 3)]).__getstate__() + restored = cls() + restored.__setstate__(state.items()) self.assertEqual(restored.getlist('a'), [5, 6]) self.assertEqual(restored.getlist('b'), [3]) diff --git a/tests/corekit/test_multidict.py b/tests/corekit/test_multidict.py index 605c32b53e..79fb37da84 100644 --- a/tests/corekit/test_multidict.py +++ b/tests/corekit/test_multidict.py @@ -88,6 +88,35 @@ def test_iter_multi_items_handles_all_mapping_shapes(self) -> None: [('a', 1), ('a', 2), ('c', 3)]) self.assertEqual(list(self.multidict.iter_multi_items([('x', 1)])), [('x', 1)]) + def test_werkzeug_31_takes_any_mapping_and_set_values(self) -> None: + import types # pylint: disable=import-outside-toplevel + + proxy = types.MappingProxyType({'a': [1, 2], 'b': {3}, 'c': set()}) + self.assertEqual(list(self.multidict.iter_multi_items(proxy)), [('a', 1), ('a', 2), ('b', 3)]) + data = self.multidict.MultiDict(proxy) + self.assertEqual(data.to_dict(flat=False), {'a': [1, 2], 'b': [3]}) + ordered = self.multidict.OrderedMultiDict(proxy) + self.assertEqual(list(ordered.items(multi=True)), [('a', 1), ('a', 2), ('b', 3)]) + + def test_werkzeug_31_or_operators(self) -> None: + for cls in (self.multidict.MultiDict, self.multidict.OrderedMultiDict): + with self.subTest(cls=cls.__name__): + data = cls([('a', 1)]) + merged = data | {'a': [2], 'b': 3} + self.assertIsInstance(merged, cls) + self.assertEqual(list(merged.items(multi=True)), [('a', 1), ('a', 2), ('b', 3)]) + self.assertEqual(list(data.items(multi=True)), [('a', 1)]) + + with self.assertRaises(TypeError): + data | [('b', 3)] # pylint: disable=pointless-statement + self.assertIs(data.__or__([('b', 3)]), NotImplemented) + + same = data + same |= [('c', 4)] + self.assertIs(same, data) + self.assertEqual(list(data.items(multi=True)), [('a', 1), ('c', 4)]) + self.assertIs(data.__ior__(5), NotImplemented) + def test_multidict_setlistdefault_returns_internal_list(self) -> None: data = self.multidict.MultiDict() values = data.setlistdefault('letters', ['a']) From 3164cafd5ba285903b483f452e36977622c6f3ec Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 09:42:18 -0400 Subject: [PATCH 4/6] fix(corekit): keep the MultiDict port verbatim to werkzeug 3.1.9 (#1963) - drop the versionchanged notes and explanatory prose this PR added - restore werkzeug 3.1.9's docstrings, comments, overloads, annotations (as strings) and code layout for the ported classes - keep pcapkit's adaptations: dict + Generic base, MissingKeyError and UnsupportedCall, no type= on getlist, local get() and _missing, the #1295 OrderedMultiDict.deepcopy, poplist's [] default, and no OrderedMultiDict deprecation warning - keep the two GraalPy dict-building lines, with shorter comments tests/corekit passes on CPython; the multidict and #1963 modules pass on CPython and GraalPy 25.4.4. Refs #1963 --- pcapkit/corekit/multidict.py | 489 +++++++++++++++++------------------ 1 file changed, 242 insertions(+), 247 deletions(-) diff --git a/pcapkit/corekit/multidict.py b/pcapkit/corekit/multidict.py index b508f1ac32..5a68cab2df 100644 --- a/pcapkit/corekit/multidict.py +++ b/pcapkit/corekit/multidict.py @@ -6,13 +6,13 @@ :mod:`pcapkit.corekit.multidict` contains multi-mapping dictionary classes, which are used to store multiple mappings of the same key. The implementation -is ported from the `Werkzeug`_ project, as of its release 3.1.9. +is ported from `Werkzeug`_ 3.1.9, ``werkzeug/datastructures/structures.py``. """ -import collections.abc +import collections.abc as cabc import copy -from typing import TYPE_CHECKING, Generic, TypeVar, cast, overload +from typing import TYPE_CHECKING, Generic, TypeVar, overload from pcapkit.utilities.exceptions import MissingKeyError, UnsupportedCall @@ -37,25 +37,19 @@ class _omd_bucket(Generic[_KT, _VT]): - """Wraps values in the :class:`OrderedMultiDict`. - - This makes it possible to keep an order over multiple different keys. It - requires a lot of extra memory and slows down access a lit, but makes it - possible to access elements in ``O(1)`` and iterate in ``O(n)`` - + """Wraps values in the :class:`OrderedMultiDict`. This makes it + possible to keep an order over multiple different keys. It requires + a lot of extra memory and slows down access a lot, but makes it + possible to access elements in O(1) and iterate in O(n). """ - prev: 'Optional[_omd_bucket]' - next: 'Optional[_omd_bucket]' - key: '_KT' - value: '_VT' - __slots__ = ('prev', 'key', 'value', 'next') + __slots__ = ("prev", "key", "value", "next") - def __init__(self, omd: 'OrderedMultiDict', key: '_KT', value: '_VT') -> 'None': - self.prev = omd._last_bucket - self.key = key - self.value = value - self.next = None + def __init__(self, omd: 'OrderedMultiDict[_KT, _VT]', key: '_KT', value: '_VT') -> 'None': + self.prev: 'Optional[_omd_bucket[_KT, _VT]]' = omd._last_bucket + self.key: '_KT' = key + self.value: '_VT' = value + self.next: 'Optional[_omd_bucket[_KT, _VT]]' = None if omd._first_bucket is None: omd._first_bucket = self @@ -63,7 +57,7 @@ def __init__(self, omd: 'OrderedMultiDict', key: '_KT', value: '_VT') -> 'None': omd._last_bucket.next = self omd._last_bucket = self - def unlink(self, omd: 'OrderedMultiDict') -> 'None': + def unlink(self, omd: 'OrderedMultiDict[_KT, _VT]') -> 'None': if self.prev: self.prev.next = self.next if self.next: @@ -92,16 +86,10 @@ def __reduce__(self) -> 'str': def iter_multi_items(mapping: 'MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'Iterator[tuple[_KT, _VT]]': # pylint: disable=line-too-long """Iterates over the items of a mapping yielding keys and values without dropping any from more complex structures. - - .. versionchanged:: 1.5.0 - - Takes any :class:`~collections.abc.Mapping`, not only a :obj:`dict`, and - yields each value of a :obj:`set` value, as Werkzeug 3.1 does. - """ if isinstance(mapping, MultiDict): yield from mapping.items(multi=True) - elif isinstance(mapping, collections.abc.Mapping): + elif isinstance(mapping, cabc.Mapping): for key, value in mapping.items(): if isinstance(value, (list, tuple, set)): for v in value: @@ -119,44 +107,48 @@ def iter_multi_items(mapping: 'MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT class MultiDict(dict, Generic[_KT, _VT]): """A :class:`MultiDict` is a dictionary subclass customized to deal with - multiple values for the same key. + multiple values for the same key which is for example used by the parsing + functions in the wrappers. This is necessary because some HTML form + elements pass multiple values for the same key. - :class:`MultiDict` implements all standard dictionary methods. Internally, - it saves all values for a key as a list, but the standard :obj:`dict` + :class:`MultiDict` implements all standard dictionary methods. + Internally, it saves all values for a key as a list, but the standard dict access methods will only return the first value for a key. If you want to - gain access to the other values, too, you have to use the :meth:`getlist` - and similar methods. - - Args: - mapping: The initial value for the :class:`MultiDict`. Either a - :class:`MultiDict`, a regular :class:`~collections.abc.Mapping`, an - iterable of ``(key, value)`` tuples, or :obj:`None`. - - It behaves like a normal :obj:`dict` thus all :obj:`dict` functions will - only return the first value when multiple values for one key are found. - - See Also: - The class is ported from the `Werkzeug`_ project, release 3.1.9 (cf. - ``werkzeug.datastructures.MultiDict``). - - .. versionchanged:: 1.5.0 - - Implements the ``|`` and ``|=`` operators, takes any - :class:`~collections.abc.Mapping` and :obj:`set` values, and calls - :obj:`dict` through :func:`super`, as Werkzeug 3.1 does. - + gain access to the other values, too, you have to use the `list` methods as + explained below. + + Basic Usage: + + >>> d = MultiDict([('a', 'b'), ('a', 'c')]) + >>> d + MultiDict([('a', 'b'), ('a', 'c')]) + >>> d['a'] + 'b' + >>> d.getlist('a') + ['b', 'c'] + >>> 'a' in d + True + + It behaves like a normal dict thus all dict functions will only return the + first value when multiple values for one key are found. + + A :class:`MultiDict` can be constructed from an iterable of + ``(key, value)`` tuples, a dict, a :class:`MultiDict` or from Werkzeug 0.2 + onwards some keyword parameters. + + :param mapping: the initial value for the :class:`MultiDict`. Either a + regular dict, an iterable of ``(key, value)`` tuples + or `None`. """ def __init__(self, mapping: 'Optional[MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long if mapping is None: super().__init__() elif isinstance(mapping, MultiDict): - # NOTE: a dict, as the other branches pass, where Werkzeug passes pairs. - # GraalPy's ``dict.__init__`` stores pairs through the overridden - # ``__setitem__``, which wraps each list in another (:issue:`1963`); a - # dict it copies as is. CPython stores either as is. + # NOTE: a dict, not pairs: GraalPy's dict.__init__ stores pairs + # through __setitem__, wrapping each list in another (:issue:`1963`). super().__init__({k: vs[:] for k, vs in mapping.lists()}) - elif isinstance(mapping, collections.abc.Mapping): + elif isinstance(mapping, cabc.Mapping): tmp = {} # type: dict[_KT, list[_VT]] for key, value in mapping.items(): if isinstance(value, (list, tuple, set)): @@ -174,57 +166,50 @@ def __init__(self, mapping: 'Optional[MultiDict[_KT, _VT] | Mapping[_KT, _VT | l tmp.setdefault(key, []).append(value) super().__init__(tmp) - def __getstate__(self) -> 'dict[_KT, list[_VT]]': + def __getstate__(self) -> 'Any': return dict(self.lists()) - def __setstate__(self, value: 'Mapping[_KT, list[_VT]] | Iterable[tuple[_KT, list[_VT]]]') -> 'None': + def __setstate__(self, value: 'Any') -> 'None': super().clear() - # NOTE: made a dict first, where Werkzeug passes ``value`` on as is: given - # pairs, GraalPy's ``dict.update`` stores them through the overridden - # ``__setitem__``, as its ``dict.__init__`` does (:issue:`1963`). + # NOTE: a dict, not pairs, as in __init__ (GraalPy, :issue:`1963`). super().update(dict(value)) - def __iter__(self) -> 'Iterator[_KT]': - # NOTE: https://github.com/python/cpython/issues/87412 -- without this - # override, ``dict(md)`` takes the value lists directly instead of calling - # :meth:`__getitem__` for the first value of each key. + def __iter__(self) -> 'Iterator[_KT]': # pylint: disable=useless-parent-delegation + # https://github.com/python/cpython/issues/87412 + # If __iter__ is not overridden, Python uses a fast path for dict(md), + # taking the data directly and getting lists of values, rather than + # calling __getitem__ and getting only the first value. return super().__iter__() def __getitem__(self, key: '_KT') -> '_VT': - """Return the first data value for this key. - - Args: - key: The key to be looked up. - - Raises: - KeyError: if the key does not exist. + """Return the first data value for this key; + raises KeyError if not found. + :param key: The key to be looked up. + :raise KeyError: if the key does not exist. """ + if key in self: lst = super().__getitem__(key) if len(lst) > 0: - return lst[0] + return lst[0] # type: ignore[no-any-return] raise MissingKeyError(key, quiet=True) def __setitem__(self, key: '_KT', value: '_VT') -> 'None': """Like :meth:`add` but removes an existing key first. - Args: - key: The key for the value. - value: The value to set. - + :param key: the key for the value. + :param value: the value to set. """ super().__setitem__(key, [value]) def add(self, key: '_KT', value: '_VT') -> 'None': """Adds a new value for the key. - Args: - key: The key for the value. - value: The value to add. - + :param key: the key for the value. + :param value: the value to add. """ - super().setdefault(key, []).append(value) # type: ignore[arg-type,attr-defined] + super().setdefault(key, []).append(value) @overload def get(self, key: '_KT') -> '_VT | _T': ... @@ -238,17 +223,11 @@ def get(self, key: '_KT', default: '_VT | _T' = None) -> '_VT | _T': # type: ig return default def getlist(self, key: '_KT') -> 'list[_VT]': - """Return the list of items for a given key. - - If that key is not in the :class:`MultiDict`, the return value - will be an empty list. - - Args: - key: The key to be looked up. - - Returns: - A :obj:`list` of all the values for the key. + """Return the list of items for a given key. If that key is not in the + `MultiDict`, the return value will be an empty list. + :param key: The key to be looked up. + :return: a :class:`list` of all the values for the key. """ try: rv = super().__getitem__(key) # type: list[_VT] @@ -257,66 +236,72 @@ def getlist(self, key: '_KT') -> 'list[_VT]': return list(rv) def setlist(self, key: '_KT', new_list: 'Iterable[_VT]') -> 'None': - """Remove the old values for a key and add new ones. - - Notes: - The list you pass the values in will be shallow-copied before it is - inserted in the dictionary. - - Args: - key: The key for which the values are set. - new_list: An iterable with the new values for the key. Old values - are removed first. - + """Remove the old values for a key and add new ones. Note that the list + you pass the values in will be shallow-copied before it is inserted in + the dictionary. + + >>> d = MultiDict() + >>> d.setlist('foo', ['1', '2']) + >>> d['foo'] + '1' + >>> d.getlist('foo') + ['1', '2'] + + :param key: The key for which the values are set. + :param new_list: An iterable with the new values for the key. Old values + are removed first. """ super().__setitem__(key, list(new_list)) - def setdefault(self, key: '_KT', default: 'Optional[_VT]' = None) -> '_VT': - """If key is in the dictionary, returns its value. - - If not, set it to default and return default. - - Args: - key: The key to be looked up. - default: The value to be set. + @overload # type: ignore[override] + def setdefault(self, key: '_KT') -> 'None': ... + @overload + def setdefault(self, key: '_KT', default: '_VT') -> '_VT': ... - Returns: - The value of the key. + def setdefault(self, key: '_KT', default: 'Optional[_VT]' = None) -> 'Optional[_VT]': + """Returns the value for the key if it is in the dict, otherwise it + returns `default` and sets that value for `key`. + :param key: The key to be looked up. + :param default: The default value to be returned if the key is not + in the dict. If not further specified it's `None`. """ if key not in self: - self[key] = cast('_VT', default) + self[key] = default # type: ignore[assignment] return self[key] def setlistdefault(self, key: '_KT', default_list: 'Optional[Iterable[_VT]]' = None) -> 'list[_VT]': - """Like :meth:`setdefault` but sets multiple values. - - The list returned is not a copy, but the list that is actually used - internally. This means that you can put new values into the - :class:`dict ` by appending items to the list. - - Args: - key: The key to be looked up. - default_list: An iterable of default values. It is either copied - (in case it was a :obj:`list`) or converted into a :obj:`list` - before returned. - + """Like `setdefault` but sets multiple values. The list returned + is not a copy, but the list that is actually used internally. This + means that you can put new values into the dict by appending items + to the list: + + >>> d = MultiDict({"foo": 1}) + >>> d.setlistdefault("foo").extend([2, 3]) + >>> d.getlist("foo") + [1, 2, 3] + + :param key: The key to be looked up. + :param default_list: An iterable of default values. It is either copied + (in case it was a list) or converted into a list + before returned. + :return: a :class:`list` """ if key not in self: super().__setitem__(key, list(default_list or ())) - return cast('list[_VT]', super().__getitem__(key)) + return super().__getitem__(key) # type: ignore[no-any-return] def items(self, multi: 'bool' = False) -> 'Iterable[tuple[_KT, _VT]]': # type: ignore[override] - """Return an interator of ``(key, value)`` paris. - - Args: - multi: If set to :obj:`True` the iterator returned will have a pair - for each value of each key. Otherwise it will only contain - pairs for the first value of each key. + """Return an iterator of ``(key, value)`` pairs. + :param multi: If set to `True` the iterator returned will have a pair + for each value of each key. Otherwise it will only + contain pairs for the first value of each key. """ + values: 'list[_VT]' + for key, values in super().items(): if multi: for value in values: @@ -324,29 +309,38 @@ def items(self, multi: 'bool' = False) -> 'Iterable[tuple[_KT, _VT]]': # type: else: yield key, values[0] - def lists(self) -> 'Iterator[tuple[_KT, list[_VT]]]': - """Return an iterator of ``(key, values)`` pairs, where ``values`` is - the :obj:`list` of all values associated with the key.""" + def lists(self) -> 'Iterable[tuple[_KT, list[_VT]]]': + """Return a iterator of ``(key, values)`` pairs, where values is the list + of all values associated with the key.""" + values: 'list[_VT]' + for key, values in super().items(): yield key, list(values) - def values(self) -> 'Iterator[_VT]': # type: ignore[override] + def values(self) -> 'Iterable[_VT]': # type: ignore[override] """Returns an iterator of the first value on every key's value list.""" + values: 'list[_VT]' + for values in super().values(): yield values[0] - def listvalues(self) -> 'Iterator[list[_VT]]': - """Return an iterator of all values associated with a key. Zipping - :meth:`keys` and this is the same as calling :meth:`lists`.""" - return super().values() # type: ignore[return-value] + def listvalues(self) -> 'Iterable[list[_VT]]': + """Return an iterator of all values associated with a key. Zipping + :meth:`keys` and this is the same as calling :meth:`lists`: + + >>> d = MultiDict({"foo": [1, 2, 3]}) + >>> zip(d.keys(), d.listvalues()) == d.lists() + True + """ + return super().values() - def copy(self) -> 'MultiDict[_KT, _VT]': + def copy(self) -> 'Self': """Return a shallow copy of this object.""" return self.__class__(self) - def deepcopy(self, memo: 'Optional[dict]' = None) -> 'MultiDict[_KT, _VT]': + def deepcopy(self, memo: 'Any' = None) -> 'Self': """Return a deep copy of this object.""" - return self.__class__(copy.deepcopy(self.to_dict(flat=False), memo=memo)) + return self.__class__(copy.deepcopy(self.to_dict(flat=False), memo)) @overload def to_dict(self, flat: 'Literal[True]' = ...) -> 'dict[_KT, _VT]': ... @@ -354,40 +348,42 @@ def to_dict(self, flat: 'Literal[True]' = ...) -> 'dict[_KT, _VT]': ... def to_dict(self, flat: 'Literal[False]') -> 'dict[_KT, list[_VT]]': ... def to_dict(self, flat: 'bool' = True) -> 'dict[_KT, _VT] | dict[_KT, list[_VT]]': - """Return the contents as regular :obj:`dict`. - - If ``flat`` is :obj:`True` the returned :obj:`dict` will only have the - first item present, if ``flat`` is :obj:`False` all values will be - returned as lists. - - Args: - flat: If set to :obj:`False` the :obj:`dict` returned will have - lists with all the values in it. Otherwise it will only contain - the first value for each key. - + """Return the contents as regular dict. If `flat` is `True` the + returned dict will only have the first item present, if `flat` is + `False` all values will be returned as lists. + + :param flat: If set to `False` the dict returned will have lists + with all the values in it. Otherwise it will only + contain the first value for each key. + :return: a :class:`dict` """ if flat: return dict(self.items()) return dict(self.lists()) def update(self, mapping: 'MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] # pylint: disable=line-too-long - """:meth:`update` extends rather than replaces existing key lists. - - If the value :obj:`list` for a key in ``other_dict`` is empty, no new - values will be added to the :obj:`dict` and the key will not be - created. - - Args: - mapping: The extending value for the :class:`MultiDict`. Either a - :class:`MultiDict`, a regular :class:`~collections.abc.Mapping`, - or an iterable of ``(key, value)`` tuples. - + """update() extends rather than replaces existing key lists: + + >>> a = MultiDict({'x': 1}) + >>> b = MultiDict({'x': 2, 'y': 3}) + >>> a.update(b) + >>> a + MultiDict([('y', 3), ('x', 1), ('x', 2)]) + + If the value list for a key in ``other_dict`` is empty, no new values + will be added to the dict and the key will not be created: + + >>> x = {'empty_list': []} + >>> y = MultiDict() + >>> y.update(x) + >>> y + MultiDict([]) """ for key, value in iter_multi_items(mapping): self.add(key, value) def __or__(self, other: 'Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]]') -> 'MultiDict[_KT, _VT]': # type: ignore[override] # pylint: disable=line-too-long - if not isinstance(other, collections.abc.Mapping): + if not isinstance(other, cabc.Mapping): return NotImplemented rv = self.copy() @@ -395,7 +391,7 @@ def __or__(self, other: 'Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_V return rv def __ior__(self, other: 'Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'Self': # type: ignore[override,misc] # pylint: disable=line-too-long - if not isinstance(other, (collections.abc.Mapping, collections.abc.Iterable)): + if not isinstance(other, (cabc.Mapping, cabc.Iterable)): return NotImplemented self.update(other) @@ -404,20 +400,26 @@ def __ior__(self, other: 'Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_ @overload def pop(self, key: '_KT') -> '_VT': ... @overload - def pop(self, key: '_KT', default: '_VT | _T' = ...) -> '_VT | _T': ... + def pop(self, key: '_KT', default: '_VT') -> '_VT': ... # pylint: disable=arguments-differ + @overload + def pop(self, key: '_KT', default: '_T') -> '_VT | _T': ... # pylint: disable=arguments-differ def pop(self, key: '_KT', default: '_VT | _T' = _missing) -> '_VT | _T': # type: ignore[assignment,misc] - """Pop the first item for a :obj:`list` on the :obj:`dict`. - - Afterwards the ``key`` is removed from the :obj:`dict`, so additional - values are discarded. - - Args: - key: The key to pop. - default: If provided the value to return if the key was not in the - dictionary. - + """Pop the first item for a list on the dict. Afterwards the + key is removed from the dict, so additional values are discarded: + + >>> d = MultiDict({"foo": [1, 2, 3]}) + >>> d.pop("foo") + 1 + >>> "foo" in d + False + + :param key: the key to pop. + :param default: if provided the value to return if the key was + not in the dictionary. """ + lst: 'list[_VT]' + try: lst = super().pop(key) @@ -432,85 +434,66 @@ def pop(self, key: '_KT', default: '_VT | _T' = _missing) -> '_VT | _T': # type raise MissingKeyError(key) from None def popitem(self) -> 'tuple[_KT, _VT]': - """Pop an item from the :obj:`dict`.""" + """Pop an item from the dict.""" + item: 'tuple[_KT, list[_VT]]' + try: item = super().popitem() - if len(item[1]) == 0: # type: ignore[arg-type] + if len(item[1]) == 0: raise MissingKeyError(item[0]) - return (item[0], item[1][0]) # type: ignore[index,return-value] + return item[0], item[1][0] except KeyError as e: raise MissingKeyError(e.args[0]) from None def poplist(self, key: '_KT') -> 'list[_VT]': - """Pop the :obj:`list` for a key from the :obj:`dict`. - - If the key is not in the :obj:`dict` an empty :obj:`list` is returned. - - Notes: - If the key does no longer exist a :obj:`list` is returned instead - of raising an error. - - Args: - key: The key to pop. - + """Pop the list for a key from the dict. If the key is not in the dict + an empty list is returned. """ - return super().pop(key, []) + return super().pop(key, []) # type: ignore[no-any-return] def popitemlist(self) -> 'tuple[_KT, list[_VT]]': - """Pop a ``(key, list)`` :obj:`tuple` from the :obj:`dict`.""" + """Pop a ``(key, list)`` tuple from the dict.""" try: - return super().popitem() # type: ignore[return-value] + return super().popitem() except KeyError as e: raise MissingKeyError(e.args[0]) from None - def __copy__(self) -> 'MultiDict[_KT, _VT]': + def __copy__(self) -> 'Self': return self.copy() - def __deepcopy__(self, memo: 'Optional[dict]' = None) -> 'MultiDict[_KT, _VT]': + def __deepcopy__(self, memo: 'Any') -> 'Self': return self.deepcopy(memo=memo) def __repr__(self) -> 'str': - return f'{type(self).__name__}({list(self.items(multi=True))!r})' + return f"{type(self).__name__}({list(self.items(multi=True))!r})" class OrderedMultiDict(MultiDict[_KT, _VT]): - """Works like a regular :class:`MultiDict` but preserves the order of the - fields. - - Args: - mapping: The initial value for the :class:`MultiDict`. Either a - :class:`MultiDict`, a regular :class:`~collections.abc.Mapping`, an - iterable of ``(key, value)`` tuples, or :obj:`None`. - - To convert the ordered multi dict into a :obj:`list` you can us the - :meth:`items` method and pass it ``multi=True``. - - In general an :class:`OrderedMultiDict` is an order of magnitude slower - than a :class:`MultiDict`. + """Works like a regular :class:`MultiDict` but preserves the + order of the fields. To convert the ordered multi dict into a + list you can use the :meth:`items` method and pass it ``multi=True``. - Notes: - Due to a limitation in Python you cannot convert an ordered multi dict - into a regular :obj:`dict` by using ``dict(multidict)``. Instead you - have to use the :meth:`to_dict` method, otherwise the internal bucket - objects are exposed. + In general an :class:`OrderedMultiDict` is an order of magnitude + slower than a :class:`MultiDict`. - Werkzeug 3.1 deprecates its ``OrderedMultiDict``, to be removed in 3.2; this - port keeps the class, without the :exc:`DeprecationWarning`. + .. admonition:: note + Due to a limitation in Python you cannot convert an ordered + multi dict into a regular dict by using ``dict(multidict)``. + Instead you have to use the :meth:`to_dict` method, otherwise + the internal bucket objects are exposed. """ - _first_bucket: 'Optional[_omd_bucket[_KT, _VT]]' - _last_bucket: 'Optional[_omd_bucket[_KT, _VT]]' def __init__(self, mapping: 'Optional[MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long super().__init__() - self._first_bucket = None - self._last_bucket = None + self._first_bucket: 'Optional[_omd_bucket[_KT, _VT]]' = None + self._last_bucket: 'Optional[_omd_bucket[_KT, _VT]]' = None if mapping is not None: self.update(mapping) - def __eq__(self, other: 'Any') -> 'bool': + def __eq__(self, other: 'object') -> 'bool': if not isinstance(other, MultiDict): return NotImplemented if isinstance(other, OrderedMultiDict): @@ -554,13 +537,13 @@ def deepcopy(self, memo: 'Optional[dict]' = None) -> 'OrderedMultiDict[_KT, _VT] """ return self.__class__(copy.deepcopy(list(self.items(multi=True)), memo=memo)) - def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'tuple[type, tuple[list[tuple[_KT, _VT]]]]': + def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'Any': return type(self), (list(self.items(multi=True)),) - def __getstate__(self) -> 'list[tuple[_KT, _VT]]': # type: ignore[override] + def __getstate__(self) -> 'Any': return list(self.items(multi=True)) - def __setstate__(self, values: 'Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] + def __setstate__(self, values: 'Any') -> 'None': self.clear() for key, value in values: @@ -568,7 +551,7 @@ def __setstate__(self, values: 'Iterable[tuple[_KT, _VT]]') -> 'None': # type: def __getitem__(self, key: '_KT') -> '_VT': if key in self: - return dict.__getitem__(self, key)[0].value + return dict.__getitem__(self, key)[0].value # type: ignore[no-any-return] raise MissingKeyError(key, quiet=True) def __setitem__(self, key: '_KT', value: '_VT') -> 'None': @@ -578,31 +561,31 @@ def __setitem__(self, key: '_KT', value: '_VT') -> 'None': def __delitem__(self, key: '_KT') -> 'None': self.pop(key) - def keys(self) -> 'Iterator[_KT]': # type: ignore[override] + def keys(self) -> 'Iterable[_KT]': # type: ignore[override] return (key for key, _ in self.items()) def __iter__(self) -> 'Iterator[_KT]': return iter(self.keys()) - def values(self) -> 'Iterator[_VT]': # type: ignore[override] - return (value for _, value in self.items()) + def values(self) -> 'Iterable[_VT]': # type: ignore[override] + return (value for key, value in self.items()) - def items(self, multi: 'bool' = False) -> 'Iterator[tuple[_KT, _VT]]': # type: ignore[override] + def items(self, multi: 'bool' = False) -> 'Iterable[tuple[_KT, _VT]]': # type: ignore[override] ptr = self._first_bucket if multi: while ptr is not None: yield ptr.key, ptr.value ptr = ptr.next else: - returned_keys = set() # type: 'set[_KT]' + returned_keys = set() while ptr is not None: if ptr.key not in returned_keys: returned_keys.add(ptr.key) yield ptr.key, ptr.value ptr = ptr.next - def lists(self) -> 'Iterator[tuple[_KT, list[_VT]]]': - returned_keys = set() # type: 'set[_KT]' + def lists(self) -> 'Iterable[tuple[_KT, list[_VT]]]': + returned_keys = set() ptr = self._first_bucket while ptr is not None: if ptr.key not in returned_keys: @@ -610,14 +593,16 @@ def lists(self) -> 'Iterator[tuple[_KT, list[_VT]]]': returned_keys.add(ptr.key) ptr = ptr.next - def listvalues(self) -> 'Iterator[list[_VT]]': - for _, values in self.lists(): + def listvalues(self) -> 'Iterable[list[_VT]]': + for _key, values in self.lists(): yield values def add(self, key: '_KT', value: '_VT') -> 'None': - dict.setdefault(self, key, []).append(_omd_bucket(self, key, value)) # type: ignore[arg-type,attr-defined] + dict.setdefault(self, key, []).append(_omd_bucket(self, key, value)) # type: ignore[misc] def getlist(self, key: '_KT') -> 'list[_VT]': + rv: 'list[_omd_bucket[_KT, _VT]]' + try: rv = dict.__getitem__(self, key) except KeyError: @@ -629,15 +614,15 @@ def setlist(self, key: '_KT', new_list: 'Iterable[_VT]') -> 'None': for value in new_list: self.add(key, value) - def setlistdefault(self, key: '_KT', default_list: 'Optional[Iterable[_VT]]' = None) -> 'NoReturn': - raise UnsupportedCall('setlistdefault is unsupported for ordered multi dicts') + def setlistdefault(self, key: 'Any', default_list: 'Any' = None) -> 'NoReturn': + raise UnsupportedCall("setlistdefault is unsupported for ordered multi dicts") def update(self, mapping: 'MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] # pylint: disable=line-too-long for key, value in iter_multi_items(mapping): self.add(key, value) def poplist(self, key: '_KT') -> 'list[_VT]': - buckets = dict.pop(self, key, []) # type: list[_omd_bucket[_KT, _VT]] + buckets: 'Iterable[_omd_bucket[_KT, _VT]]' = dict.pop(self, key, []) for bucket in buckets: bucket.unlink(self) return [x.value for x in buckets] @@ -645,11 +630,15 @@ def poplist(self, key: '_KT') -> 'list[_VT]': @overload def pop(self, key: '_KT') -> '_VT': ... @overload - def pop(self, key: '_KT', default: '_VT | _T' = ...) -> '_VT | _T': ... + def pop(self, key: '_KT', default: '_VT') -> '_VT': ... # pylint: disable=arguments-differ + @overload + def pop(self, key: '_KT', default: '_T') -> '_VT | _T': ... # pylint: disable=arguments-differ def pop(self, key: '_KT', default: '_VT | _T' = _missing) -> '_VT | _T': # type: ignore[assignment,misc] + buckets: 'list[_omd_bucket[_KT, _VT]]' + try: - buckets = dict.pop(self, key) # type: list[_omd_bucket[_KT, _VT]] + buckets = dict.pop(self, key) except KeyError: if default is not _missing: return default @@ -662,10 +651,13 @@ def pop(self, key: '_KT', default: '_VT | _T' = _missing) -> '_VT | _T': # type return buckets[0].value def popitem(self) -> 'tuple[_KT, _VT]': + key: '_KT' + buckets: 'list[_omd_bucket[_KT, _VT]]' + try: - key, buckets = cast('tuple[_KT, list[_omd_bucket[_KT, _VT]]]', dict.popitem(self)) + key, buckets = dict.popitem(self) except KeyError as e: - raise MissingKeyError(str(e)) from None + raise MissingKeyError(e.args[0]) from None for bucket in buckets: bucket.unlink(self) @@ -673,10 +665,13 @@ def popitem(self) -> 'tuple[_KT, _VT]': return key, buckets[0].value def popitemlist(self) -> 'tuple[_KT, list[_VT]]': + key: '_KT' + buckets: 'list[_omd_bucket[_KT, _VT]]' + try: - key, buckets = cast('tuple[_KT, list[_omd_bucket[_KT, _VT]]]', dict.popitem(self)) + key, buckets = dict.popitem(self) except KeyError as e: - raise MissingKeyError(str(e)) from None + raise MissingKeyError(e.args[0]) from None for bucket in buckets: bucket.unlink(self) From 8f98e6b11bbfaa96e3381164398a81d6f0bbad7c Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 09:57:36 -0400 Subject: [PATCH 5/6] test(project): know the useless-parent-delegation pragma (#1963) - add useless-parent-delegation (W0246) to KNOWN_MESSAGES: the pragma on MultiDict.__iter__ keeps upstream's CPython 87412 fast-path workaround Refs #1963 --- tests/project/test_pylint_pragma_names_1923_unit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/project/test_pylint_pragma_names_1923_unit.py b/tests/project/test_pylint_pragma_names_1923_unit.py index 96646a5842..cf43ffa6a2 100644 --- a/tests/project/test_pylint_pragma_names_1923_unit.py +++ b/tests/project/test_pylint_pragma_names_1923_unit.py @@ -88,6 +88,7 @@ 'unused-argument', 'unused-import', 'unused-wildcard-import', + 'useless-parent-delegation', 'wildcard-import', 'wrong-import-position', }) From 1258482df79cd01b9ef30dd66f9148a0c8219a63 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 10:04:09 -0400 Subject: [PATCH 6/6] test(corekit): declare the GraalPy semantics module ALL-tier (#1963) - add __tier__ = 'ALL': the module pins GraalPy-only semantics, so it has to keep running on every interpreter leg once ALL becomes opt-in (#1978) Refs #1963 --- tests/corekit/test_graalpy_semantics_1963_unit.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/corekit/test_graalpy_semantics_1963_unit.py b/tests/corekit/test_graalpy_semantics_1963_unit.py index 3488d25770..8431ab6576 100644 --- a/tests/corekit/test_graalpy_semantics_1963_unit.py +++ b/tests/corekit/test_graalpy_semantics_1963_unit.py @@ -24,6 +24,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): pins GraalPy-only semantics, so it must run on every +#: interpreter leg. +__tier__ = 'ALL' + #: Octets the stream holds: more than a buffer's worth. DATA = bytes(range(64)) #: A line longer than GraalPy's shared-read threshold, then more data.