diff --git a/pcapkit/corekit/io.py b/pcapkit/corekit/io.py index ff49c090de..05df937ff8 100644 --- a/pcapkit/corekit/io.py +++ b/pcapkit/corekit/io.py @@ -286,6 +286,32 @@ def _seek_buffer(self) -> 'int': self._buffer.seek(buf_off, io.SEEK_SET) return self._buffer_cur - buf_off + def _take_buffer(self, size: 'int', /, *, line: 'bool' = False) -> 'bytes': + r"""Read up to ``size`` octets from the buffer's cursor, as a copy, and move past them. + + Args: + size: Octets to read, no more than :meth:`_seek_buffer` says the buffer can serve. + line: Stop after the first ``b'\n'``, as :meth:`io.BytesIO.readline` does. + + Returns: + The octets read. + + Note: + The octets are copied out of :attr:`_buffer_view`, not read with + :meth:`io.BytesIO.read`, ``read1`` or ``readline``. On GraalPy those can hand + back :obj:`bytes` that share the memory of a :class:`io.BytesIO` whose buffer + is exported, as this one's is, so the next :meth:`_write_buffer`, which writes + through the view, changed octets that had already been returned + (:issue:`1963`). A slice of the view, copied, is a copy on every interpreter. + + """ + buf_off = self._buffer.tell() + buf = self._buffer_view[buf_off:buf_off + size].tobytes() + if line and (eol := buf.find(b'\n')) >= 0: + buf = buf[:eol + 1] + self._buffer.seek(buf_off + len(buf), io.SEEK_SET) + return buf + def close(self) -> 'None': """Flush and close this stream. This method has no effect if the file is already closed. Once the file is closed, any operation on the file (e.g. reading or writing) will raise @@ -369,7 +395,7 @@ def readline(self, size: 'int | None' = -1, /) -> 'bytes': buf = b'' else: buf_rem = self._seek_buffer() - buf = self._buffer.readline(buf_rem if size < 0 else min(size, buf_rem)) + buf = self._take_buffer(buf_rem if size < 0 else min(size, buf_rem), line=True) # NOTE: an unbounded ``readline`` has to keep going until the line ends, and # capping it at the buffer's content -- which the line need not end inside -- @@ -651,7 +677,7 @@ def read(self, size: 'int | None' = -1, /) -> 'bytes': buf = b'' else: buf_rem = self._seek_buffer() - buf = self._buffer.read(buf_rem if size < 0 else min(size, buf_rem)) + buf = self._take_buffer(buf_rem if size < 0 else min(size, buf_rem)) size_rem = -1 if size < 0 or (size_rem := size - len(buf)) > 0: @@ -685,7 +711,7 @@ def read1(self, size: 'int | None' = -1, /) -> 'bytes': # NOTE: no continuation from the stream when the buffer answered, since # :meth:`read1` is specified to return only buffered octets if any are. buf_rem = self._seek_buffer() - buf = self._buffer.read1(buf_rem if size < 0 else min(size, buf_rem)) + buf = self._take_buffer(buf_rem if size < 0 else min(size, buf_rem)) if not buf: # pragma: no cover size_rem = -1 diff --git a/pcapkit/corekit/multidict.py b/pcapkit/corekit/multidict.py index bcf3742839..5a68cab2df 100644 --- a/pcapkit/corekit/multidict.py +++ b/pcapkit/corekit/multidict.py @@ -6,19 +6,20 @@ :mod:`pcapkit.corekit.multidict` contains multi-mapping dictionary classes, which are used to store multiple mappings of the same key. The implementation -is inspired and based on the `Werkzeug`_ project. +is ported from `Werkzeug`_ 3.1.9, ``werkzeug/datastructures/structures.py``. """ +import collections.abc as cabc import copy -from typing import TYPE_CHECKING, Generic, TypeVar, cast, overload +from typing import TYPE_CHECKING, Generic, TypeVar, overload from pcapkit.utilities.exceptions import MissingKeyError, UnsupportedCall if TYPE_CHECKING: from typing import Any, Iterable, Iterator, Mapping, NoReturn, Optional, SupportsIndex - from typing_extensions import Literal + from typing_extensions import Literal, Self __all__ = ['MultiDict', 'OrderedMultiDict'] @@ -36,25 +37,19 @@ class _omd_bucket(Generic[_KT, _VT]): - """Wraps values in the :class:`OrderedMultiDict`. - - This makes it possible to keep an order over multiple different keys. It - requires a lot of extra memory and slows down access a lit, but makes it - possible to access elements in ``O(1)`` and iterate in ``O(n)`` - + """Wraps values in the :class:`OrderedMultiDict`. This makes it + possible to keep an order over multiple different keys. It requires + a lot of extra memory and slows down access a lot, but makes it + possible to access elements in O(1) and iterate in O(n). """ - prev: 'Optional[_omd_bucket]' - next: 'Optional[_omd_bucket]' - key: '_KT' - value: '_VT' - __slots__ = ('prev', 'key', 'value', 'next') + __slots__ = ("prev", "key", "value", "next") - def __init__(self, omd: 'OrderedMultiDict', key: '_KT', value: '_VT') -> 'None': - self.prev = omd._last_bucket - self.key = key - self.value = value - self.next = None + def __init__(self, omd: 'OrderedMultiDict[_KT, _VT]', key: '_KT', value: '_VT') -> 'None': + self.prev: 'Optional[_omd_bucket[_KT, _VT]]' = omd._last_bucket + self.key: '_KT' = key + self.value: '_VT' = value + self.next: 'Optional[_omd_bucket[_KT, _VT]]' = None if omd._first_bucket is None: omd._first_bucket = self @@ -62,7 +57,7 @@ def __init__(self, omd: 'OrderedMultiDict', key: '_KT', value: '_VT') -> 'None': omd._last_bucket.next = self omd._last_bucket = self - def unlink(self, omd: 'OrderedMultiDict') -> 'None': + def unlink(self, omd: 'OrderedMultiDict[_KT, _VT]') -> 'None': if self.prev: self.prev.next = self.next if self.next: @@ -88,15 +83,15 @@ def __reduce__(self) -> 'str': ############################################################################### -def iter_multi_items(mapping: 'Mapping[_KT, _VT | Iterable[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'Iterator[tuple[_KT, _VT]]': # pylint: disable=line-too-long +def iter_multi_items(mapping: 'MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'Iterator[tuple[_KT, _VT]]': # pylint: disable=line-too-long """Iterates over the items of a mapping yielding keys and values without dropping any from more complex structures. """ if isinstance(mapping, MultiDict): yield from mapping.items(multi=True) - elif isinstance(mapping, dict): + elif isinstance(mapping, cabc.Mapping): for key, value in mapping.items(): - if isinstance(value, (tuple, list)): + if isinstance(value, (list, tuple, set)): for v in value: yield key, v else: @@ -112,93 +107,109 @@ def iter_multi_items(mapping: 'Mapping[_KT, _VT | Iterable[_VT]] | Iterable[tupl class MultiDict(dict, Generic[_KT, _VT]): """A :class:`MultiDict` is a dictionary subclass customized to deal with - multiple values for the same key. + multiple values for the same key which is for example used by the parsing + functions in the wrappers. This is necessary because some HTML form + elements pass multiple values for the same key. - :class:`MultiDict` implements all standard dictionary methods. Internally, - it saves all values for a key as a list, but the standard :obj:`dict` + :class:`MultiDict` implements all standard dictionary methods. + Internally, it saves all values for a key as a list, but the standard dict access methods will only return the first value for a key. If you want to - gain access to the other values, too, you have to use the :meth:`getlist` - and similar methods. - - Args: - mapping: The initial value for the :class:`MultiDict`. Either a - regular :obj:`dict`, an iterable of ``(key, value)`` tuples, or - :obj:`None`. - - It behaves like a normal :obj:`dict` thus all :obj:`dict` functions will - only return the first value when multiple values for one key are found. - - See Also: - The class is inspired from and based on the `Werkzeug`_ project (cf. - ``werkzeug.datastructures.MultiDict``). - + gain access to the other values, too, you have to use the `list` methods as + explained below. + + Basic Usage: + + >>> d = MultiDict([('a', 'b'), ('a', 'c')]) + >>> d + MultiDict([('a', 'b'), ('a', 'c')]) + >>> d['a'] + 'b' + >>> d.getlist('a') + ['b', 'c'] + >>> 'a' in d + True + + It behaves like a normal dict thus all dict functions will only return the + first value when multiple values for one key are found. + + A :class:`MultiDict` can be constructed from an iterable of + ``(key, value)`` tuples, a dict, a :class:`MultiDict` or from Werkzeug 0.2 + onwards some keyword parameters. + + :param mapping: the initial value for the :class:`MultiDict`. Either a + regular dict, an iterable of ``(key, value)`` tuples + or `None`. """ - def __init__(self, mapping: 'Optional[dict[_KT, _VT] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long - if isinstance(mapping, MultiDict): - dict.__init__(self, ((k, v[:]) for k, v in mapping.lists())) - elif isinstance(mapping, dict): + def __init__(self, mapping: 'Optional[MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long + if mapping is None: + super().__init__() + elif isinstance(mapping, MultiDict): + # NOTE: a dict, not pairs: GraalPy's dict.__init__ stores pairs + # through __setitem__, wrapping each list in another (:issue:`1963`). + super().__init__({k: vs[:] for k, vs in mapping.lists()}) + elif isinstance(mapping, cabc.Mapping): tmp = {} # type: dict[_KT, list[_VT]] for key, value in mapping.items(): - if isinstance(value, (tuple, list)): - if len(value) == 0: - continue + if isinstance(value, (list, tuple, set)): value = list(value) + + if not value: + continue else: value = [value] tmp[key] = value - dict.__init__(self, tmp) + super().__init__(tmp) else: tmp = {} - for key, value in mapping or (): + for key, value in mapping: # type: ignore[misc] tmp.setdefault(key, []).append(value) - dict.__init__(self, tmp) + super().__init__(tmp) - def __getstate__(self) -> 'dict[_KT, list[_VT]]': + def __getstate__(self) -> 'Any': return dict(self.lists()) - def __setstate__(self, value: 'Iterable[tuple[_KT, list[_VT]]]') -> 'None': - dict.clear(self) - dict.update(self, value) # type: ignore[arg-type] + def __setstate__(self, value: 'Any') -> 'None': + super().clear() + # NOTE: a dict, not pairs, as in __init__ (GraalPy, :issue:`1963`). + super().update(dict(value)) - def __iter__(self) -> 'Iterator[_KT]': - return dict.__iter__(self) + def __iter__(self) -> 'Iterator[_KT]': # pylint: disable=useless-parent-delegation + # https://github.com/python/cpython/issues/87412 + # If __iter__ is not overridden, Python uses a fast path for dict(md), + # taking the data directly and getting lists of values, rather than + # calling __getitem__ and getting only the first value. + return super().__iter__() def __getitem__(self, key: '_KT') -> '_VT': - """Return the first data value for this key. - - Args: - key: The key to be looked up. - - Raises: - KeyError: if the key does not exist. + """Return the first data value for this key; + raises KeyError if not found. + :param key: The key to be looked up. + :raise KeyError: if the key does not exist. """ + if key in self: - lst = dict.__getitem__(self, key) + lst = super().__getitem__(key) if len(lst) > 0: - return lst[0] + return lst[0] # type: ignore[no-any-return] raise MissingKeyError(key, quiet=True) def __setitem__(self, key: '_KT', value: '_VT') -> 'None': """Like :meth:`add` but removes an existing key first. - Args: - key: The key for the value. - value: The value to set. - + :param key: the key for the value. + :param value: the value to set. """ - dict.__setitem__(self, key, [value]) + super().__setitem__(key, [value]) def add(self, key: '_KT', value: '_VT') -> 'None': """Adds a new value for the key. - Args: - key: The key for the value. - value: The value to add. - + :param key: the key for the value. + :param value: the value to add. """ - dict.setdefault(self, key, []).append(value) # type: ignore[arg-type,attr-defined] + super().setdefault(key, []).append(value) @overload def get(self, key: '_KT') -> '_VT | _T': ... @@ -212,118 +223,124 @@ def get(self, key: '_KT', default: '_VT | _T' = None) -> '_VT | _T': # type: ig return default def getlist(self, key: '_KT') -> 'list[_VT]': - """Return the list of items for a given key. - - If that key is not in the :class:`MultiDict`, the return value - will be an empty list. - - Args: - key: The key to be looked up. - - Returns: - A :obj:`list` of all the values for the key. + """Return the list of items for a given key. If that key is not in the + `MultiDict`, the return value will be an empty list. + :param key: The key to be looked up. + :return: a :class:`list` of all the values for the key. """ try: - rv = dict.__getitem__(self, key) # type: list[_VT] + rv = super().__getitem__(key) # type: list[_VT] except KeyError: return [] return list(rv) def setlist(self, key: '_KT', new_list: 'Iterable[_VT]') -> 'None': - """Remove the old values for a key and add new ones. - - Notes: - The list you pass the values in will be shallow-copied before it is - inserted in the dictionary. - - Args: - key: The key for which the values are set. - new_list: An iterable with the new values for the key. Old values - are removed first. - + """Remove the old values for a key and add new ones. Note that the list + you pass the values in will be shallow-copied before it is inserted in + the dictionary. + + >>> d = MultiDict() + >>> d.setlist('foo', ['1', '2']) + >>> d['foo'] + '1' + >>> d.getlist('foo') + ['1', '2'] + + :param key: The key for which the values are set. + :param new_list: An iterable with the new values for the key. Old values + are removed first. """ - dict.__setitem__(self, key, list(new_list)) - - def setdefault(self, key: '_KT', default: 'Optional[_VT]' = None) -> '_VT': - """If key is in the dictionary, returns its value. + super().__setitem__(key, list(new_list)) - If not, set it to default and return default. - - Args: - key: The key to be looked up. - default: The value to be set. + @overload # type: ignore[override] + def setdefault(self, key: '_KT') -> 'None': ... + @overload + def setdefault(self, key: '_KT', default: '_VT') -> '_VT': ... - Returns: - The value of the key. + def setdefault(self, key: '_KT', default: 'Optional[_VT]' = None) -> 'Optional[_VT]': + """Returns the value for the key if it is in the dict, otherwise it + returns `default` and sets that value for `key`. + :param key: The key to be looked up. + :param default: The default value to be returned if the key is not + in the dict. If not further specified it's `None`. """ if key not in self: - self[key] = cast('_VT', default) - else: - default = self[key] - return default # type: ignore[return-value] - - def setlistdefault(self, key: '_KT', default_list: 'Optional[Iterable[_VT]]' = None) -> 'list[_VT]': - """Like :meth:`setdefault` but sets multiple values. - - The list returned is not a copy, but the list that is actually used - internally. This means that you can put new values into the - :class:`dict ` by appending items to the list. + self[key] = default # type: ignore[assignment] - Args: - key: The key to be looked up. - default_list: An iterable of default values. It is either copied - (in case it was a :obj:`list`) or converted into a :obj:`list` - before returned. + return self[key] + def setlistdefault(self, key: '_KT', default_list: 'Optional[Iterable[_VT]]' = None) -> 'list[_VT]': + """Like `setdefault` but sets multiple values. The list returned + is not a copy, but the list that is actually used internally. This + means that you can put new values into the dict by appending items + to the list: + + >>> d = MultiDict({"foo": 1}) + >>> d.setlistdefault("foo").extend([2, 3]) + >>> d.getlist("foo") + [1, 2, 3] + + :param key: The key to be looked up. + :param default_list: An iterable of default values. It is either copied + (in case it was a list) or converted into a list + before returned. + :return: a :class:`list` """ if key not in self: - default_list = list(default_list or ()) - dict.__setitem__(self, key, default_list) - else: - default_list = cast('list[_VT]', dict.__getitem__(self, key)) - return default_list + super().__setitem__(key, list(default_list or ())) - def items(self, multi: 'bool' = False) -> 'Iterable[tuple[_KT, _VT]]': # type: ignore[override] - """Return an interator of ``(key, value)`` paris. + return super().__getitem__(key) # type: ignore[no-any-return] - Args: - multi: If set to :obj:`True` the iterator returned will have a pair - for each value of each key. Otherwise it will only contain - pairs for the first value of each key. + def items(self, multi: 'bool' = False) -> 'Iterable[tuple[_KT, _VT]]': # type: ignore[override] + """Return an iterator of ``(key, value)`` pairs. + :param multi: If set to `True` the iterator returned will have a pair + for each value of each key. Otherwise it will only + contain pairs for the first value of each key. """ - for key, values in dict.items(self): + values: 'list[_VT]' + + for key, values in super().items(): if multi: for value in values: yield key, value else: yield key, values[0] - def lists(self) -> 'Iterator[tuple[_KT, list[_VT]]]': - """Return an iterator of ``(key, values)`` pairs, where ``values`` is - the :obj:`list` of all values associated with the key.""" - for key, values in dict.items(self): + def lists(self) -> 'Iterable[tuple[_KT, list[_VT]]]': + """Return a iterator of ``(key, values)`` pairs, where values is the list + of all values associated with the key.""" + values: 'list[_VT]' + + for key, values in super().items(): yield key, list(values) - def values(self) -> 'Iterator[_VT]': # type: ignore[override] + def values(self) -> 'Iterable[_VT]': # type: ignore[override] """Returns an iterator of the first value on every key's value list.""" - for values in dict.values(self): + values: 'list[_VT]' + + for values in super().values(): yield values[0] - def listvalues(self) -> 'Iterator[list[_VT]]': - """Return an iterator of all values associated with a key. Zipping - :meth:`keys` and this is the same as calling :meth:`lists`.""" - return dict.values(self) # type: ignore[return-value] + def listvalues(self) -> 'Iterable[list[_VT]]': + """Return an iterator of all values associated with a key. Zipping + :meth:`keys` and this is the same as calling :meth:`lists`: - def copy(self) -> 'MultiDict[_KT, _VT]': + >>> d = MultiDict({"foo": [1, 2, 3]}) + >>> zip(d.keys(), d.listvalues()) == d.lists() + True + """ + return super().values() + + def copy(self) -> 'Self': """Return a shallow copy of this object.""" return self.__class__(self) - def deepcopy(self, memo: 'Optional[dict]' = None) -> 'MultiDict[_KT, _VT]': + def deepcopy(self, memo: 'Any' = None) -> 'Self': """Return a deep copy of this object.""" - return self.__class__(copy.deepcopy(self.to_dict(flat=False), memo=memo)) # type: ignore[arg-type] + return self.__class__(copy.deepcopy(self.to_dict(flat=False), memo)) @overload def to_dict(self, flat: 'Literal[True]' = ...) -> 'dict[_KT, _VT]': ... @@ -331,57 +348,80 @@ def to_dict(self, flat: 'Literal[True]' = ...) -> 'dict[_KT, _VT]': ... def to_dict(self, flat: 'Literal[False]') -> 'dict[_KT, list[_VT]]': ... def to_dict(self, flat: 'bool' = True) -> 'dict[_KT, _VT] | dict[_KT, list[_VT]]': - """Return the contents as regular :obj:`dict`. - - If ``flat`` is :obj:`True` the returned :obj:`dict` will only have the - first item present, if ``flat`` is :obj:`False` all values will be - returned as lists. - - Args: - flat: If set to :obj:`False` the :obj:`dict` returned will have - lists with all the values in it. Otherwise it will only contain - the first value for each key. - + """Return the contents as regular dict. If `flat` is `True` the + returned dict will only have the first item present, if `flat` is + `False` all values will be returned as lists. + + :param flat: If set to `False` the dict returned will have lists + with all the values in it. Otherwise it will only + contain the first value for each key. + :return: a :class:`dict` """ if flat: return dict(self.items()) return dict(self.lists()) - def update(self, mapping: 'Mapping[_KT, _VT] | Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] - """:meth:`update` extends rather than replaces existing key lists. + def update(self, mapping: 'MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] # pylint: disable=line-too-long + """update() extends rather than replaces existing key lists: - If the value :obj:`list` for a key in ``other_dict`` is empty, no new - values will be added to the :obj:`dict` and the key will not be - created. + >>> a = MultiDict({'x': 1}) + >>> b = MultiDict({'x': 2, 'y': 3}) + >>> a.update(b) + >>> a + MultiDict([('y', 3), ('x', 1), ('x', 2)]) - Args: - mapping: The extending value for the :class:`MultiDict`. Either a - regular :obj:`dict`, an iterable of ``(key, value)`` tuples, or - :obj:`None`. + If the value list for a key in ``other_dict`` is empty, no new values + will be added to the dict and the key will not be created: + >>> x = {'empty_list': []} + >>> y = MultiDict() + >>> y.update(x) + >>> y + MultiDict([]) """ for key, value in iter_multi_items(mapping): - MultiDict.add(self, key, value) + self.add(key, value) + + def __or__(self, other: 'Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]]') -> 'MultiDict[_KT, _VT]': # type: ignore[override] # pylint: disable=line-too-long + if not isinstance(other, cabc.Mapping): + return NotImplemented + + rv = self.copy() + rv.update(other) + return rv + + def __ior__(self, other: 'Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'Self': # type: ignore[override,misc] # pylint: disable=line-too-long + if not isinstance(other, (cabc.Mapping, cabc.Iterable)): + return NotImplemented + + self.update(other) + return self @overload def pop(self, key: '_KT') -> '_VT': ... @overload - def pop(self, key: '_KT', default: '_VT | _T' = ...) -> '_VT | _T': ... + def pop(self, key: '_KT', default: '_VT') -> '_VT': ... # pylint: disable=arguments-differ + @overload + def pop(self, key: '_KT', default: '_T') -> '_VT | _T': ... # pylint: disable=arguments-differ def pop(self, key: '_KT', default: '_VT | _T' = _missing) -> '_VT | _T': # type: ignore[assignment,misc] - """Pop the first item for a :obj:`list` on the :obj:`dict`. - - Afterwards the ``key`` is removed from the :obj:`dict`, so additional - values are discarded. - - Args: - key: The key to pop. - default: If provided the value to return if the key was not in the - dictionary. - + """Pop the first item for a list on the dict. Afterwards the + key is removed from the dict, so additional values are discarded: + + >>> d = MultiDict({"foo": [1, 2, 3]}) + >>> d.pop("foo") + 1 + >>> "foo" in d + False + + :param key: the key to pop. + :param default: if provided the value to return if the key was + not in the dictionary. """ + lst: 'list[_VT]' + try: - lst = dict.pop(self, key) + lst = super().pop(key) if len(lst) == 0: raise MissingKeyError(key) @@ -394,81 +434,66 @@ def pop(self, key: '_KT', default: '_VT | _T' = _missing) -> '_VT | _T': # type raise MissingKeyError(key) from None def popitem(self) -> 'tuple[_KT, _VT]': - """Pop an item from the :obj:`dict`.""" + """Pop an item from the dict.""" + item: 'tuple[_KT, list[_VT]]' + try: - item = dict.popitem(self) + item = super().popitem() - if len(item[1]) == 0: # type: ignore[arg-type] + if len(item[1]) == 0: raise MissingKeyError(item[0]) - return (item[0], item[1][0]) # type: ignore[index,return-value] + return item[0], item[1][0] except KeyError as e: raise MissingKeyError(e.args[0]) from None def poplist(self, key: '_KT') -> 'list[_VT]': - """Pop the :obj:`list` for a key from the :obj:`dict`. - - If the key is not in the :obj:`dict` an empty :obj:`list` is returned. - - Notes: - If the key does no longer exist a :obj:`list` is returned instead - of raising an error. - - Args: - key: The key to pop. - + """Pop the list for a key from the dict. If the key is not in the dict + an empty list is returned. """ - return dict.pop(self, key, []) + return super().pop(key, []) # type: ignore[no-any-return] def popitemlist(self) -> 'tuple[_KT, list[_VT]]': - """Pop a ``(key, list)`` :obj:`tuple` from the :obj:`dict`.""" + """Pop a ``(key, list)`` tuple from the dict.""" try: - return dict.popitem(self) # type: ignore[return-value] + return super().popitem() except KeyError as e: raise MissingKeyError(e.args[0]) from None - def __copy__(self) -> 'MultiDict[_KT, _VT]': + def __copy__(self) -> 'Self': return self.copy() - def __deepcopy__(self, memo: 'Optional[dict]' = None) -> 'MultiDict[_KT, _VT]': + def __deepcopy__(self, memo: 'Any') -> 'Self': return self.deepcopy(memo=memo) def __repr__(self) -> 'str': - return f'{type(self).__name__}({list(self.items(multi=True))!r})' + return f"{type(self).__name__}({list(self.items(multi=True))!r})" class OrderedMultiDict(MultiDict[_KT, _VT]): - """Works like a regular :class:`MultiDict` but preserves the order of the - fields. + """Works like a regular :class:`MultiDict` but preserves the + order of the fields. To convert the ordered multi dict into a + list you can use the :meth:`items` method and pass it ``multi=True``. - Args: - mapping: The initial value for the :class:`MultiDict`. Either a - regular :obj:`dict`, an iterable of ``(key, value)`` tuples, or - :obj:`None`. + In general an :class:`OrderedMultiDict` is an order of magnitude + slower than a :class:`MultiDict`. - To convert the ordered multi dict into a :obj:`list` you can us the - :meth:`items` method and pass it ``multi=True``. - - In general an :class:`OrderedMultiDict` is an order of magnitude slower - than a :class:`MultiDict`. - - Notes: - Due to a limitation in Python you cannot convert an ordered multi dict - into a regular :obj:`dict` by using ``dict(multidict)``. Instead you - have to use the :meth:`to_dict` method, otherwise the internal bucket - objects are exposed. + .. admonition:: note + Due to a limitation in Python you cannot convert an ordered + multi dict into a regular dict by using ``dict(multidict)``. + Instead you have to use the :meth:`to_dict` method, otherwise + the internal bucket objects are exposed. """ - _first_bucket: 'Optional[_omd_bucket[_KT, _VT]]' - _last_bucket: 'Optional[_omd_bucket[_KT, _VT]]' - def __init__(self, mapping: 'Optional[dict[_KT, _VT] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long - dict.__init__(self) # pylint: disable=non-parent-init-called - self._first_bucket = self._last_bucket = None + def __init__(self, mapping: 'Optional[MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]]' = None) -> 'None': # pylint: disable=line-too-long + super().__init__() + self._first_bucket: 'Optional[_omd_bucket[_KT, _VT]]' = None + self._last_bucket: 'Optional[_omd_bucket[_KT, _VT]]' = None if mapping is not None: - OrderedMultiDict.update(self, mapping) + self.update(mapping) - def __eq__(self, other: 'Any') -> 'bool': + def __eq__(self, other: 'object') -> 'bool': if not isinstance(other, MultiDict): return NotImplemented if isinstance(other, OrderedMultiDict): @@ -512,20 +537,21 @@ def deepcopy(self, memo: 'Optional[dict]' = None) -> 'OrderedMultiDict[_KT, _VT] """ return self.__class__(copy.deepcopy(list(self.items(multi=True)), memo=memo)) - def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'tuple[type, tuple[list[tuple[_KT, _VT]]]]': + def __reduce_ex__(self, protocol: 'SupportsIndex') -> 'Any': return type(self), (list(self.items(multi=True)),) - def __getstate__(self) -> 'list[tuple[_KT, _VT]]': # type: ignore[override] + def __getstate__(self) -> 'Any': return list(self.items(multi=True)) - def __setstate__(self, values: 'Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] - dict.clear(self) + def __setstate__(self, values: 'Any') -> 'None': + self.clear() + for key, value in values: self.add(key, value) def __getitem__(self, key: '_KT') -> '_VT': if key in self: - return dict.__getitem__(self, key)[0].value + return dict.__getitem__(self, key)[0].value # type: ignore[no-any-return] raise MissingKeyError(key, quiet=True) def __setitem__(self, key: '_KT', value: '_VT') -> 'None': @@ -535,31 +561,31 @@ def __setitem__(self, key: '_KT', value: '_VT') -> 'None': def __delitem__(self, key: '_KT') -> 'None': self.pop(key) - def keys(self) -> 'Iterator[_KT]': # type: ignore[override] + def keys(self) -> 'Iterable[_KT]': # type: ignore[override] return (key for key, _ in self.items()) def __iter__(self) -> 'Iterator[_KT]': return iter(self.keys()) - def values(self) -> 'Iterator[_VT]': # type: ignore[override] - return (value for _, value in self.items()) + def values(self) -> 'Iterable[_VT]': # type: ignore[override] + return (value for key, value in self.items()) - def items(self, multi: 'bool' = False) -> 'Iterator[tuple[_KT, _VT]]': # type: ignore[override] + def items(self, multi: 'bool' = False) -> 'Iterable[tuple[_KT, _VT]]': # type: ignore[override] ptr = self._first_bucket if multi: while ptr is not None: yield ptr.key, ptr.value ptr = ptr.next else: - returned_keys = set() # type: 'set[_KT]' + returned_keys = set() while ptr is not None: if ptr.key not in returned_keys: returned_keys.add(ptr.key) yield ptr.key, ptr.value ptr = ptr.next - def lists(self) -> 'Iterator[tuple[_KT, list[_VT]]]': - returned_keys = set() # type: 'set[_KT]' + def lists(self) -> 'Iterable[tuple[_KT, list[_VT]]]': + returned_keys = set() ptr = self._first_bucket while ptr is not None: if ptr.key not in returned_keys: @@ -567,14 +593,16 @@ def lists(self) -> 'Iterator[tuple[_KT, list[_VT]]]': returned_keys.add(ptr.key) ptr = ptr.next - def listvalues(self) -> 'Iterator[list[_VT]]': - for _, values in self.lists(): + def listvalues(self) -> 'Iterable[list[_VT]]': + for _key, values in self.lists(): yield values def add(self, key: '_KT', value: '_VT') -> 'None': - dict.setdefault(self, key, []).append(_omd_bucket(self, key, value)) # type: ignore[arg-type,attr-defined] + dict.setdefault(self, key, []).append(_omd_bucket(self, key, value)) # type: ignore[misc] def getlist(self, key: '_KT') -> 'list[_VT]': + rv: 'list[_omd_bucket[_KT, _VT]]' + try: rv = dict.__getitem__(self, key) except KeyError: @@ -586,15 +614,15 @@ def setlist(self, key: '_KT', new_list: 'Iterable[_VT]') -> 'None': for value in new_list: self.add(key, value) - def setlistdefault(self, key: '_KT', default_list: 'Optional[Iterable[_VT]]' = None) -> 'NoReturn': - raise UnsupportedCall('setlistdefault is unsupported for ordered multi dicts') + def setlistdefault(self, key: 'Any', default_list: 'Any' = None) -> 'NoReturn': + raise UnsupportedCall("setlistdefault is unsupported for ordered multi dicts") - def update(self, mapping: 'Mapping[_KT, _VT] | Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] + def update(self, mapping: 'MultiDict[_KT, _VT] | Mapping[_KT, _VT | list[_VT] | tuple[_VT, ...] | set[_VT]] | Iterable[tuple[_KT, _VT]]') -> 'None': # type: ignore[override] # pylint: disable=line-too-long for key, value in iter_multi_items(mapping): - OrderedMultiDict.add(self, key, value) + self.add(key, value) def poplist(self, key: '_KT') -> 'list[_VT]': - buckets = dict.pop(self, key, []) # type: list[_omd_bucket[_KT, _VT]] + buckets: 'Iterable[_omd_bucket[_KT, _VT]]' = dict.pop(self, key, []) for bucket in buckets: bucket.unlink(self) return [x.value for x in buckets] @@ -602,11 +630,15 @@ def poplist(self, key: '_KT') -> 'list[_VT]': @overload def pop(self, key: '_KT') -> '_VT': ... @overload - def pop(self, key: '_KT', default: '_VT | _T' = ...) -> '_VT | _T': ... + def pop(self, key: '_KT', default: '_VT') -> '_VT': ... # pylint: disable=arguments-differ + @overload + def pop(self, key: '_KT', default: '_T') -> '_VT | _T': ... # pylint: disable=arguments-differ def pop(self, key: '_KT', default: '_VT | _T' = _missing) -> '_VT | _T': # type: ignore[assignment,misc] + buckets: 'list[_omd_bucket[_KT, _VT]]' + try: - buckets = dict.pop(self, key) # type: list[_omd_bucket[_KT, _VT]] + buckets = dict.pop(self, key) except KeyError: if default is not _missing: return default @@ -619,10 +651,13 @@ def pop(self, key: '_KT', default: '_VT | _T' = _missing) -> '_VT | _T': # type return buckets[0].value def popitem(self) -> 'tuple[_KT, _VT]': + key: '_KT' + buckets: 'list[_omd_bucket[_KT, _VT]]' + try: - key, buckets = cast('tuple[_KT, list[_omd_bucket[_KT, _VT]]]', dict.popitem(self)) + key, buckets = dict.popitem(self) except KeyError as e: - raise MissingKeyError(str(e)) from None + raise MissingKeyError(e.args[0]) from None for bucket in buckets: bucket.unlink(self) @@ -630,10 +665,13 @@ def popitem(self) -> 'tuple[_KT, _VT]': return key, buckets[0].value def popitemlist(self) -> 'tuple[_KT, list[_VT]]': + key: '_KT' + buckets: 'list[_omd_bucket[_KT, _VT]]' + try: - key, buckets = cast('tuple[_KT, list[_omd_bucket[_KT, _VT]]]', dict.popitem(self)) + key, buckets = dict.popitem(self) except KeyError as e: - raise MissingKeyError(str(e)) from None + raise MissingKeyError(e.args[0]) from None for bucket in buckets: bucket.unlink(self) diff --git a/tests/corekit/test_fields_field.py b/tests/corekit/test_fields_field.py index d79722f0a0..4258370c92 100644 --- a/tests/corekit/test_fields_field.py +++ b/tests/corekit/test_fields_field.py @@ -1,6 +1,7 @@ from __future__ import annotations import struct +import sys import threading import unittest @@ -173,7 +174,9 @@ def test_wire_declared_length_far_larger_than_the_buffer_is_rejected_without_all in case a future regression reintroduces the allocation instead of just moving the threshold. """ - huge_declared_length = 2 ** 34 # 16 GiB; must never actually be allocated. + # 16 GiB; must never actually be allocated. ``struct`` refuses a count + # past ``sys.maxsize``, which is 2 GiB less one on GraalPy (#1963). + huge_declared_length = min(2 ** 34, sys.maxsize) field = self.BytesField(length=huge_declared_length) with time_limit(): diff --git a/tests/corekit/test_graalpy_semantics_1963_unit.py b/tests/corekit/test_graalpy_semantics_1963_unit.py new file mode 100644 index 0000000000..8431ab6576 --- /dev/null +++ b/tests/corekit/test_graalpy_semantics_1963_unit.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +"""GraalPy behaviours :mod:`pcapkit.corekit` no longer depends on (#1963). + +Each was measured on GraalPy 25.4 and does not happen on CPython, so each class +here stands in for GraalPy's behaviour and checks the module does not lean on +CPython's: + +* :class:`io.BytesIO` ``read``, ``read1`` and ``readline`` from the start of an + exported buffer can hand back :obj:`bytes` sharing its memory, so + :class:`~pcapkit.corekit.io.SeekableReader` returned octets that its next write + to the buffer then changed. +* ``dict.__init__`` and ``dict.update`` given ``(key, value)`` pairs store them + through a subclass's ``__setitem__``, so ``MultiDict(MultiDict)``, and a + ``MultiDict`` restored from pairs of its state, wrapped every value list in another. + +On GraalPy itself, ``test_end_of_stream_reads_ahead_only_as_far_as_asked`` in +``test_io_seekable_reader_short_reads_1586_unit.py`` and the ``MultiDict`` copies +in ``test_multidict.py`` fail without the fixes. + +""" + +import io +import unittest + +from tests._support import reimport_once_per_class + +#: Test tier (#1958): pins GraalPy-only semantics, so it must run on every +#: interpreter leg. +__tier__ = 'ALL' + +#: Octets the stream holds: more than a buffer's worth. +DATA = bytes(range(64)) +#: A line longer than GraalPy's shared-read threshold, then more data. +LINES = b'x' * 12 + b'\n' + bytes(range(32)) + + +class AliasingBytesIO(io.BytesIO): + """A buffer whose reads return a view of its own memory, as GraalPy's can. + + On GraalPy, once :meth:`getbuffer` has exported the buffer, a read of more than + eight octets from its start returns :obj:`bytes` that a later write through the + export changes. A :class:`memoryview` of the same octets does the same here. + + """ + + def _view(self, size: 'int | None') -> 'memoryview': + start = self.tell() + view = self.getbuffer() + end = len(view) if size is None or size < 0 else min(start + size, len(view)) + self.seek(end) + return view[start:end] + + def read(self, size: 'int | None' = -1, /) -> 'memoryview': # type: ignore[override] + return self._view(size) + + def read1(self, size: 'int | None' = -1, /) -> 'memoryview': # type: ignore[override] + return self._view(size) + + def readline(self, size: 'int | None' = -1, /) -> 'memoryview': # type: ignore[override] + start = self.tell() + view = self._view(size) + eol = view.tobytes().find(b'\n') + if eol < 0: + return view + self.seek(start + eol + 1) + return view[:eol + 1] + + +class SeekableReaderCopiesItsBufferTests(unittest.TestCase): + """Octets the reader returned stay as they were when it slides its buffer.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def reader(self, data: 'bytes') -> 'object': + from pcapkit.corekit.io import SeekableReader # pylint: disable=import-outside-toplevel + + reader = SeekableReader(io.BytesIO(data), buffer_size=16, stream_closing=False) + # pylint: disable=protected-access + reader._buffer_view.release() + reader._buffer = AliasingBytesIO(bytearray(reader._buffer_size)) + reader._buffer_view = reader._buffer.getbuffer() + self.addCleanup(reader.close) + return reader + + def check(self, method: 'str', data: 'bytes', size: 'int') -> None: + reader = self.reader(data) + self.assertEqual(reader.read(16), data[:16]) + self.assertEqual(reader.seek(0), 0) + + got = getattr(reader, method)(size) + self.assertEqual(got, data[:len(got)]) + self.assertGreater(len(got), 8) + + # the next read from the stream slides the buffer over every octet in it + self.assertEqual(reader.seek(16), 16) + self.assertEqual(reader.read(32), data[16:48]) + self.assertEqual(got, data[:len(got)]) + self.assertIsInstance(got, bytes) + + def test_read(self) -> None: + self.check('read', DATA, 10) + + def test_read1(self) -> None: + self.check('read1', DATA, 10) + + def test_readline(self) -> None: + self.check('readline', LINES, -1) + + +class PairsThroughSetitem(dict): + """:class:`dict`, with GraalPy's ``__init__`` and ``update``: pairs go through ``__setitem__``. + + :class:`~pcapkit.corekit.multidict.MultiDict` reaches :obj:`dict` through + :func:`super`, as Werkzeug's does, so a class deriving from it and then from + this one puts this in its place, as GraalPy's :obj:`dict` would be. + + """ + + def __init__(self, *args: 'object', **kwargs: 'object') -> None: # pylint: disable=super-init-not-called + self._store(dict.__init__, args, kwargs) + + def update(self, *args: 'object', **kwargs: 'object') -> None: + self._store(dict.update, args, kwargs) + + def _store(self, method: 'object', args: 'tuple', kwargs: 'dict') -> None: + """Pairs through ``self.__setitem__``, anything else as :class:`dict` stores it.""" + if args and not isinstance(args[0], dict): + method(self, **kwargs) # type: ignore[operator] + for key, value in args[0]: + self[key] = value + return + method(self, *args, **kwargs) # type: ignore[operator] + + +class MultiDictCopyTests(unittest.TestCase): + """``MultiDict(MultiDict)`` keeps each key's values as they were.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def multidict(self) -> 'type': + """A :class:`~pcapkit.corekit.multidict.MultiDict` over :class:`PairsThroughSetitem`.""" + from pcapkit.corekit.multidict import MultiDict # pylint: disable=import-outside-toplevel + + class GraalMultiDict(MultiDict, PairsThroughSetitem): # type: ignore[misc] + """A :class:`~pcapkit.corekit.multidict.MultiDict` on GraalPy's :obj:`dict`.""" + + return GraalMultiDict + + def test_the_stand_in_wraps_pairs_as_graalpy_does(self) -> None: + """A guard on the stand-in, not on the fix: it passes with or without it. + + The tests below fail without the fix only because the stand-in stores + pairs through ``__setitem__``, as GraalPy does, and a plain dict as is. + Without this, a stand-in that lost either would let them pass vacuously. + + """ + cls = self.multidict() + # MultiDict's ``super()`` reaches the stand-in, not :obj:`dict` + self.assertEqual([c.__name__ for c in cls.__mro__[1:4]], + ['MultiDict', 'PairsThroughSetitem', 'dict']) + + for method in (PairsThroughSetitem.__init__, PairsThroughSetitem.update): + with self.subTest(method=method.__name__): + data = cls() + method(data, [('a', [5, 6])]) + self.assertEqual(dict.__getitem__(data, 'a'), [[5, 6]]) + method(data, {'b': [7]}) + self.assertEqual(dict.__getitem__(data, 'b'), [7]) + + def test_a_copy_keeps_the_value_lists(self) -> None: + cls = self.multidict() + data = cls([('a', 5), ('a', 6), ('b', 3)]) + copied = cls(data) + self.assertEqual(copied.getlist('a'), [5, 6]) + self.assertEqual(copied.getlist('b'), [3]) + + def test_state_restored_from_pairs_keeps_the_value_lists(self) -> None: + cls = self.multidict() + state = cls([('a', 5), ('a', 6), ('b', 3)]).__getstate__() + restored = cls() + restored.__setstate__(state.items()) + self.assertEqual(restored.getlist('a'), [5, 6]) + self.assertEqual(restored.getlist('b'), [3]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/corekit/test_multidict.py b/tests/corekit/test_multidict.py index 605c32b53e..79fb37da84 100644 --- a/tests/corekit/test_multidict.py +++ b/tests/corekit/test_multidict.py @@ -88,6 +88,35 @@ def test_iter_multi_items_handles_all_mapping_shapes(self) -> None: [('a', 1), ('a', 2), ('c', 3)]) self.assertEqual(list(self.multidict.iter_multi_items([('x', 1)])), [('x', 1)]) + def test_werkzeug_31_takes_any_mapping_and_set_values(self) -> None: + import types # pylint: disable=import-outside-toplevel + + proxy = types.MappingProxyType({'a': [1, 2], 'b': {3}, 'c': set()}) + self.assertEqual(list(self.multidict.iter_multi_items(proxy)), [('a', 1), ('a', 2), ('b', 3)]) + data = self.multidict.MultiDict(proxy) + self.assertEqual(data.to_dict(flat=False), {'a': [1, 2], 'b': [3]}) + ordered = self.multidict.OrderedMultiDict(proxy) + self.assertEqual(list(ordered.items(multi=True)), [('a', 1), ('a', 2), ('b', 3)]) + + def test_werkzeug_31_or_operators(self) -> None: + for cls in (self.multidict.MultiDict, self.multidict.OrderedMultiDict): + with self.subTest(cls=cls.__name__): + data = cls([('a', 1)]) + merged = data | {'a': [2], 'b': 3} + self.assertIsInstance(merged, cls) + self.assertEqual(list(merged.items(multi=True)), [('a', 1), ('a', 2), ('b', 3)]) + self.assertEqual(list(data.items(multi=True)), [('a', 1)]) + + with self.assertRaises(TypeError): + data | [('b', 3)] # pylint: disable=pointless-statement + self.assertIs(data.__or__([('b', 3)]), NotImplemented) + + same = data + same |= [('c', 4)] + self.assertIs(same, data) + self.assertEqual(list(data.items(multi=True)), [('a', 1), ('c', 4)]) + self.assertIs(data.__ior__(5), NotImplemented) + def test_multidict_setlistdefault_returns_internal_list(self) -> None: data = self.multidict.MultiDict() values = data.setlistdefault('letters', ['a']) diff --git a/tests/corekit/test_sentinel_singleton_1842_unit.py b/tests/corekit/test_sentinel_singleton_1842_unit.py index 2c8c81ca18..fd7ceecc32 100644 --- a/tests/corekit/test_sentinel_singleton_1842_unit.py +++ b/tests/corekit/test_sentinel_singleton_1842_unit.py @@ -33,6 +33,7 @@ import importlib import inspect import pickle +import sys import unittest from typing import TYPE_CHECKING @@ -57,6 +58,11 @@ #: Python's own singletons, which a sentinel is to behave like. BUILTINS = (None, Ellipsis, NotImplemented) +#: The lowest :mod:`pickle` protocol a :class:`typing.Generic` subclass round-trips at. +#: GraalPy's :class:`typing.Generic` is not a heap type, so ``copyreg._reduce_ex``, +#: which protocols 0 and 1 use, stops at it and fails on ``Generic(obj)`` (#1963). +LOWEST_PROTOCOL = 2 if sys.implementation.name == 'graalpy' else 0 + #: Pickles written at ``75a9f237b``, keyed by ``(object name, protocol)``. ``NULL`` #: reduced to the module-level ``_get_null``; the others took the default reduce, #: which is ``copyreg._reconstructor`` below protocol 2 and ``NEWOBJ`` from 2 on. @@ -440,7 +446,7 @@ def test_a_parsed_mpl_seed_keeps_its_identity(self) -> 'None': bad = [] if copy.deepcopy(schema).options[0].seed is not no_value: bad.append('deepcopy') - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + for protocol in range(LOWEST_PROTOCOL, pickle.HIGHEST_PROTOCOL + 1): if pickle.loads(pickle.dumps(schema, protocol=protocol)).options[0].seed is not no_value: bad.append(protocol) self.assertEqual(bad, []) diff --git a/tests/dumpkit/test_fallback_cycle_unit.py b/tests/dumpkit/test_fallback_cycle_unit.py index 13d4cea895..3ec2a7790d 100644 --- a/tests/dumpkit/test_fallback_cycle_unit.py +++ b/tests/dumpkit/test_fallback_cycle_unit.py @@ -19,6 +19,7 @@ import json import os import plistlib +import sys import tempfile import unittest from unittest import mock @@ -53,7 +54,9 @@ def setUp(self) -> None: reimport_once_per_class(self) self.tmpdir = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with self.addCleanup(self.tmpdir.cleanup) - if resource is not None: + # NOTE: GraalPy's ``resource`` has no ``setrlimit`` (#1963), so there, as + # off POSIX, the file is left to the leg's own cap. + if resource is not None and sys.implementation.name != 'graalpy': # NOTE: without the fix, the class case writes without bound, so # cap the file size rather than let it fill the disk. limits = resource.getrlimit(resource.RLIMIT_FSIZE) diff --git a/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py b/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py index 7ee7930e38..8caf566405 100644 --- a/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py +++ b/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py @@ -21,6 +21,7 @@ import io import os import struct +import sys import tempfile import unittest import warnings @@ -110,6 +111,9 @@ def test_a_pcap_record_cut_at_eof_is_traced_as_the_octets_it_has(self) -> None: """#1541: a record claiming more than the file holds writes what it holds.""" for claim in CLAIMS: with self.subTest(claim=claim): + if 16 + claim > sys.maxsize: + self.skipTest('the frame reader asks the file for the 16-octet header and the claim at once, ' + 'past sys.maxsize on GraalPy (#1963)') self.check(self.trace(_cut_pcap(claim)), _frames()[-1]) def test_a_pcapng_captured_length_past_its_block_is_traced_as_the_octets_it_has(self) -> None: diff --git a/tests/foundation/test_extraction_no_eof.py b/tests/foundation/test_extraction_no_eof.py index 6b66bbd08f..9df6a21600 100644 --- a/tests/foundation/test_extraction_no_eof.py +++ b/tests/foundation/test_extraction_no_eof.py @@ -126,11 +126,17 @@ #: "the extraction did not finish" from "the child failed for some other reason". SENTINEL = 'NO-EOF-RETURNED' +#: How a child caps its address space. GraalPy's ``resource`` has no ``setrlimit`` +#: (#1963); a child there is held by the heap cap the leg gives every GraalPy +#: process, in ``GRAAL_PYTHON_VM_ARGS``, which it inherits. +CHILD_CAP = ('' if sys.implementation.name == 'graalpy' + else 'resource.setrlimit(resource.RLIMIT_AS, ({cap}, {cap}))') + #: Preamble for every child: cap the address space, put the tree under test first #: on ``sys.path``, and *prove* that is the tree which got imported. CHILD_PREAMBLE = """ import resource, sys, warnings -resource.setrlimit(resource.RLIMIT_AS, ({cap}, {cap})) +""" + CHILD_CAP + """ sys.path.insert(0, {root!r}) warnings.simplefilter('ignore') import pcapkit diff --git a/tests/project/test_pylint_pragma_names_1923_unit.py b/tests/project/test_pylint_pragma_names_1923_unit.py index 96646a5842..cf43ffa6a2 100644 --- a/tests/project/test_pylint_pragma_names_1923_unit.py +++ b/tests/project/test_pylint_pragma_names_1923_unit.py @@ -88,6 +88,7 @@ 'unused-argument', 'unused-import', 'unused-wildcard-import', + 'useless-parent-delegation', 'wildcard-import', 'wrong-import-position', }) diff --git a/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py b/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py index b000b5541b..404ac34ab7 100644 --- a/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py +++ b/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py @@ -24,6 +24,7 @@ import pickle import plistlib import struct +import sys import tempfile import unittest import warnings @@ -33,6 +34,11 @@ RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) +#: The lowest :mod:`pickle` protocol a :class:`typing.Generic` subclass round-trips at. +#: GraalPy's :class:`typing.Generic` is not a heap type, so ``copyreg._reduce_ex``, +#: which protocols 0 and 1 use, stops at it and fails on ``Generic(obj)`` (#1963). +LOWEST_PROTOCOL = 2 if sys.implementation.name == 'graalpy' else 0 + #: NTP seconds and fraction of the Mobility Message Replay Protection option: #: the UNIX epoch counted from 1900, plus a quarter of a second. NTP = (0x83aa7e80, 0x40000000) @@ -163,7 +169,7 @@ def test_pickling_round_trips(self) -> None: from pcapkit.corekit.infoclass import Info _, mesg_id, timestamp = self.parse() - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + for protocol in range(LOWEST_PROTOCOL, pickle.HIGHEST_PROTOCOL + 1): for value in (mesg_id.ntp_timestamp, timestamp.pmip_timestamp, mesg_id, timestamp): with self.subTest(protocol=protocol, type=type(value).__name__): restored = pickle.loads(pickle.dumps(value, protocol)) diff --git a/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py b/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py index 4248096bf0..5e575a6a80 100644 --- a/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py +++ b/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py @@ -15,6 +15,7 @@ import io import struct +import sys import unittest import warnings @@ -54,6 +55,9 @@ def test_overrun_warns_and_is_kept_as_captured(self) -> None: header = Header(GLOBAL).info for claim in (61, 1000, 0x7FFF_FFFF): with self.subTest(claim=claim): + if 16 + claim > sys.maxsize: + self.skipTest('the frame reader asks the file for the 16-octet header and the claim at once, ' + 'past sys.maxsize on GraalPy (#1963)') octets = _record(claim) frame, messages = self.parse(octets) self.assertIn(f'PCAP: [Frame 4] captured length {claim} runs past the data, ' diff --git a/tests/protocols/misc/test_pcapng_unit.py b/tests/protocols/misc/test_pcapng_unit.py index a5a5243026..a2fa28f7bc 100644 --- a/tests/protocols/misc/test_pcapng_unit.py +++ b/tests/protocols/misc/test_pcapng_unit.py @@ -4756,6 +4756,8 @@ def test_the_captured_len_vector_parses_under_a_memory_cap(self) -> None: """ if importlib.util.find_spec('resource') is None: # pragma: no cover self.skipTest('the resource module is unavailable on this platform') + if sys.implementation.name == 'graalpy': # pragma: no cover + self.skipTest('GraalPy has no resource.setrlimit to cap the subprocess with (#1963)') source = textwrap.dedent(''' import io, json, resource, struct, sys, warnings diff --git a/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py b/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py index 9f20c72c2a..bcb8ac5d6e 100644 --- a/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py +++ b/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import sys import unittest from collections import ChainMap from collections.abc import Mapping @@ -10,6 +11,11 @@ RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) +#: GraalPy's :mod:`abc` is ``_py_abc``, which keeps its registry and caches as separate +#: attributes, not in the C accelerator's ``_abc_impl`` these tests inspect (#1963). +NO_ABC_IMPL = sys.implementation.name == 'graalpy' +NO_ABC_IMPL_REASON = "GraalPy's abc is _py_abc, which has no _abc_impl (#1963)" + @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') class SchemaMetaAbcCacheTests(unittest.TestCase): @@ -109,6 +115,7 @@ def test_asking_schema_first_does_not_break_mapping_for_a_plain_dict(self) -> No # THEN ask Mapping. A plain dict must still be reported as one. self.assertTrue(isinstance({}, Mapping)) + @unittest.skipIf(NO_ABC_IMPL, NO_ABC_IMPL_REASON) def test_schema_subclasses_have_their_own_abc_impl(self) -> None: """Every schema class gets its own ABCMeta registry/cache triple. @@ -177,6 +184,7 @@ class _SharedChild(_Shared): del _SharedChild._abc_impl return _Shared, _SharedChild + @unittest.skipIf(NO_ABC_IMPL, NO_ABC_IMPL_REASON) def test_probing_the_same_exact_type_reproduces_439s_shape(self) -> None: """The general mechanism, generalised beyond :class:`dict` and ``Schema``. @@ -197,6 +205,7 @@ def test_probing_the_same_exact_type_reproduces_439s_shape(self) -> None: self.assertTrue(isinstance(ChainMap(), Mapping)) self.assertTrue(isinstance(ChainMap(), _SharedChild)) + @unittest.skipIf(NO_ABC_IMPL, NO_ABC_IMPL_REASON) def test_probing_a_different_exact_type_gives_a_false_negative(self) -> None: """A pitfall in reproducing #439: the cache is keyed by *exact* type.