From b0d32247ce19ce1f3017d443fd6bdeab998fddbb Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sat, 10 Oct 2026 13:30:45 -0400 Subject: [PATCH] fix(protocols): bound a frame's dissection at 64 layers (#1610) - Each layer dissects the next from inside its own parse, so deep nesting of any protocol, such as IP in IP or stacked 802.1Q tags, hit the recursion limit, and beholder left the rest raw, silently; #1604's bound restarts at every tunnel. FRAME_LAYER_LIMIT (64) now caps the layers ProtocolBase._parse_next_layer dissects in one frame: past it the rest stays Raw, with one ProtocolWarning. - IPv6 counts the extension headers it walks as layers, as the protocol chain does, and stops its walk at the bound; what a walked ESP decrypts counts on after them. - _make_payload builds each payload one layer further in, so from_data of a cut frame re-parses no deeper than the parse did, where it went past the recursion limit before, and warns of the bound once, not once a layer. - tcp_segment read a TCP 65th layer back out of the Raw the bound leaves, as it does a header the parser rejected (#1518); that Raw is now marked _past_layer_limit, and is not read. - The PEP records recursion with a depth limit as a decision open to revisit. --- docs/source/contributing/pep.rst | 27 ++ docs/source/pcapkit/protocols/protocol.rst | 22 + pcapkit/protocols/internet/ipv6.py | 27 +- pcapkit/protocols/protocol.py | 109 ++++- pcapkit/toolkit/pcap.py | 7 +- .../test_frame_layer_limit_1610_unit.py | 397 ++++++++++++++++++ 6 files changed, 574 insertions(+), 15 deletions(-) create mode 100644 tests/protocols/test_frame_layer_limit_1610_unit.py diff --git a/docs/source/contributing/pep.rst b/docs/source/contributing/pep.rst index 1e638ccef3..5406e2c4ba 100644 --- a/docs/source/contributing/pep.rst +++ b/docs/source/contributing/pep.rst @@ -417,6 +417,33 @@ Two traps for anyone benchmarking this library. ``reassembly=True`` and only the switch measures nothing; and timing several capture shapes in one process inflates them by up to 73%, so each shape wants its own interpreter. +Recursive Dissection, With a Depth Limit +---------------------------------------- + +**Decided for now, open to revisit.** Each layer dissects the next from inside +its own parse, so dissecting a frame recurses once per layer, at 9 to 12 Python +frames a layer. Deep nesting -- IP in IP, stacked 802.1Q tags, long chains of +IPv6 extension headers -- used to exhaust the recursion limit and leave the rest +of the frame raw, silently, at a depth that varied with the interpreter and the +caller's stack (:issue:`1604`, :issue:`1610`). + +Recursion stays, with a counted bound: +:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` caps a frame at 64 layers, +and :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` caps one +chain of extension headers at 32. Past either, the rest stays +:class:`~pcapkit.protocols.misc.raw.Raw`, with one +:exc:`~pcapkit.utilities.warnings.ProtocolWarning`. Raising +:func:`sys.setrecursionlimit` instead was rejected: it is process-wide, and the +cut-off would still move with the caller's stack. The bound fits with room to +spare, but not unlimited room. Under :program:`pytest` at the default limit of +1,000, the costliest 64-layer frame needs a limit of 834 on Python 3.11 and 756 +on 3.14. That frame is IPv4 carrying a chain of 32 extension headers, then IPv4 +in IPv4 carrying 28 more, then TCP. + +An iterative model would need no such bound: a loop that dissects one layer and +takes the next from it. It changes how every protocol hands over its payload, +so it is a design of its own, and may replace the recursion in a future release. + Logging Integration ------------------- diff --git a/docs/source/pcapkit/protocols/protocol.rst b/docs/source/pcapkit/protocols/protocol.rst index b6dc2dab1a..43a96eba65 100644 --- a/docs/source/pcapkit/protocols/protocol.rst +++ b/docs/source/pcapkit/protocols/protocol.rst @@ -73,6 +73,7 @@ Internal Definitions below. .. automethod:: _lookup_next_layer .. automethod:: _decode_next_layer .. automethod:: _import_next_layer + .. automethod:: _parse_next_layer .. automethod:: _get_context .. autoattribute:: _data @@ -82,6 +83,7 @@ Internal Definitions below. .. autoattribute:: _protos .. autoattribute:: _seekset .. autoattribute:: _sigterm + .. autoattribute:: _past_layer_limit .. autoattribute:: __data__ .. automethod:: __init__ @@ -99,6 +101,26 @@ Internal Definitions below. .. autoattribute:: _exproto .. autoattribute:: _exctx +Layers per Frame +---------------- + +A frame is dissected to at most +:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of its protocol +chain, counted from the first layer the frame record carries, through tunnels, +VLAN tags and IPv6 extension headers alike. What follows is kept as +:class:`~pcapkit.protocols.misc.raw.Raw`, with one +:exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1610`), and is +marked :attr:`~pcapkit.protocols.protocol.Protocol._past_layer_limit`, so the +default engine's TCP reassembly and flow tracing do not read a TCP segment +there back out, as they do one the TCP parser rejected (:issue:`1518`). A +protocol constructed from its own octets is not counted itself, so the count +starts at the first layer it dissects. One chain of extension headers is also bounded on +its own, by :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT`. +The ``dpkt``, ``scapy``, ``pyshark`` and ``pypcapfile`` engines dissect with +their own parsers, which do not apply this bound (:issue:`1609`). + +.. autodata:: pcapkit.protocols.protocol.FRAME_LAYER_LIMIT + Data Models ----------- diff --git a/pcapkit/protocols/internet/ipv6.py b/pcapkit/protocols/internet/ipv6.py index 16cb16bd05..66b163dfe1 100644 --- a/pcapkit/protocols/internet/ipv6.py +++ b/pcapkit/protocols/internet/ipv6.py @@ -37,6 +37,7 @@ from pcapkit.protocols.data.internet.ipv6 import IPv6 as Data_IPv6 from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT from pcapkit.protocols.internet.ip import IP +from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT, _frame_depth from pcapkit.protocols.schema.internet.ipv6 import IPv6 as Schema_IPv6 from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length from pcapkit.utilities.decorators import beholder @@ -535,6 +536,7 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, _exthdr = [] # (parser class, info) per extension header split = None # type: Optional[tuple[int, int]] # at first Fragment header opaque = False # walk ended at a non-zero Fragment Offset + depth = _frame_depth.get() # layers of the frame up to this one # traverse if next header is an extension header payload = self.__header__.get_payload() @@ -553,14 +555,27 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, if len(_protos) >= EXTENSION_HEADER_LIMIT: break + # NOTE: Likewise past :data:`FRAME_LAYER_LIMIT + # ` layers of the + # frame (:issue:`1610`), since each header walked is one of them. + if depth + len(_protos) >= FRAME_LAYER_LIMIT: + break + # # directly break when No Next Header occurs # if proto.name == 'IPv6-NoNxt': # proto = None # break # make protocol name - next_ = self._import_next_layer(proto, packet=packet, version=6, extension=True, - payload=payload) # type: ignore[misc,call-arg,arg-type] + # NOTE: Counted after every header walked, as it would be were each + # dissected by the one before it. ESP dissects its decrypted + # payload even when walked, and those layers count on from here. + token = _frame_depth.set(depth + len(_protos)) + try: + next_ = self._import_next_layer(proto, packet=packet, version=6, extension=True, + payload=payload) # type: ignore[misc,call-arg,arg-type] + finally: + _frame_depth.reset(token) info = next_.info # A layer with no ``next`` field cannot safely continue the walk. @@ -657,8 +672,12 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, # ``fragment`` above, which this leaves as it was. ipv6_exthdr = ProtoChain.from_list(_protos) # type: ignore[arg-type] self._exthdr_depth = len(_protos) # the headers the upper layer follows - return super()._decode_next_layer(ipv6, None if opaque else proto, raw_len, packet=packet, - ipv6_exthdr=ipv6_exthdr, payload=payload) + token = _frame_depth.set(depth + len(_protos)) + try: + return super()._decode_next_layer(ipv6, None if opaque else proto, raw_len, packet=packet, + ipv6_exthdr=ipv6_exthdr, payload=payload) + finally: + _frame_depth.reset(token) @beholder # type: ignore[arg-type] def _import_next_layer(self, proto: 'int', length: 'Optional[int]' = None, *, # pylint: disable=arguments-differ diff --git a/pcapkit/protocols/protocol.py b/pcapkit/protocols/protocol.py index db087f35a5..a600d6a5bb 100644 --- a/pcapkit/protocols/protocol.py +++ b/pcapkit/protocols/protocol.py @@ -16,6 +16,7 @@ import collections import collections.abc import contextlib +import contextvars import difflib import enum import functools @@ -49,8 +50,9 @@ from pcapkit.utilities.decorators import beholder, seekset from pcapkit.utilities.exceptions import (BaseError, ProtocolError, ProtocolNotFound, ProtocolNotImplemented, RegistryError, StructError, - UnsupportedCall) -from pcapkit.utilities.warnings import RegistryWarning, UnknownFieldWarning, hold_warnings, warn + UnsupportedCall, stacklevel) +from pcapkit.utilities.warnings import (ProtocolWarning, RegistryWarning, UnknownFieldWarning, + hold_warnings, warn) if TYPE_CHECKING: from enum import IntEnum as StdlibEnum @@ -389,6 +391,37 @@ def _check_construction_keywords(cls: 'type', kwargs: 'dict[str, Any]', UnknownFieldWarning) +#: Most layers dissected in one frame (:issue:`1610`). A layer is one entry of +#: the frame's protocol chain, counted from the first layer the frame record +#: carries, usually its link layer; a protocol constructed from its own octets +#: is not counted itself, so the count starts at the first layer it dissects. +#: IPv6 extension headers are layers like any other, whether +#: :class:`~pcapkit.protocols.internet.ipv6.IPv6` walks them or each dissects +#: the next. What follows the last layer is kept as +#: :class:`~pcapkit.protocols.misc.raw.Raw`, with one +#: :exc:`~pcapkit.utilities.warnings.ProtocolWarning`, unless it would have been +#: left raw anyway or there is nothing left. +#: +#: Each layer dissects the next from inside its own parse, so without the bound +#: a deep enough frame, such as IP in IP or stacked 802.1Q tags, stops wherever +#: Python's recursion limit falls, which depends on the interpreter and on how +#: deep the caller's own stack is. A tunnel does not restart the count, and nor +#: does the end of a chain of extension headers, which +#: :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` bounds on +#: its own. +FRAME_LAYER_LIMIT = 64 + +#: contextvars.ContextVar[int]: How many layers of the frame being dissected +#: come before the next one :meth:`ProtocolBase._parse_next_layer` parses, c.f. +#: :data:`FRAME_LAYER_LIMIT`. +_frame_depth = contextvars.ContextVar('_frame_depth', default=0) + +#: contextvars.ContextVar[bool]: Whether a payload is being rebuilt by +#: :meth:`ProtocolBase._make_payload`, so that the layer it is rebuilt for +#: parses the same octets again, c.f. :data:`FRAME_LAYER_LIMIT`. +_rebuilding = contextvars.ContextVar('_rebuilding', default=False) + + def _short_of_hint(klass: 'Type[ProtocolBase]', captured: 'int') -> 'bool': """Tell whether ``captured`` octets may be too few for a ``klass`` header. @@ -536,6 +569,13 @@ class ProtocolBase(Generic[_PT, _ST], metaclass=ProtocolMeta): #: ``object.__new__(SomeProtocol)``) sees, so reading it is always safe. _exctx: 'Optional[ContextRegistry]' = None + #: Whether this layer is past :data:`FRAME_LAYER_LIMIT`, and so was kept as + #: :class:`~pcapkit.protocols.misc.raw.Raw` by :meth:`_parse_next_layer` + #: without being parsed. Such a layer carries an ``error`` as one that + #: failed to parse does, so this is what tells the two apart: what follows + #: the limit is not to be read on (:issue:`1610`). + _past_layer_limit: 'bool' = False + ########################################################################## # Properties. ########################################################################## @@ -2017,6 +2057,14 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc neither. The payload is then found from the structure of ``data`` instead, by :meth:`_lookup_payload`. + The payload is built one layer further into the frame than this + protocol, as :meth:`_parse_next_layer` counts it, since each layer built + parses its own octets again. So rebuilding a frame stops at + :data:`FRAME_LAYER_LIMIT` where parsing it did, rather than that many + layers below each layer of it (:issue:`1610`). Nor does the payload's + parse warn of the limit: this protocol's own parse goes over the same + octets, so the rebuilt frame warns of it once, as the parsed one did. + Args: data: protocol data exclude: keys of ``data`` that are not the payload, though they may @@ -2032,7 +2080,12 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc from pcapkit.protocols.misc.null import \ NoPayload # pylint: disable=import-outside-toplevel return NoPayload() - return proto.from_data(data[name]) + token, rebuilding = _frame_depth.set(_frame_depth.get() + 1), _rebuilding.set(True) + try: + return proto.from_data(data[name]) + finally: + _rebuilding.reset(rebuilding) + _frame_depth.reset(token) proto = cast('Optional[Type[Protocol]]', data.get('__next_type__')) if proto is None or not (isinstance(proto, type) and issubclass(proto, ProtocolBase)): @@ -2046,7 +2099,12 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc NoPayload # pylint: disable=import-outside-toplevel return NoPayload() - return proto.from_data(data[name]) + token, rebuilding = _frame_depth.set(_frame_depth.get() + 1), _rebuilding.set(True) + try: + return proto.from_data(data[name]) + finally: + _rebuilding.reset(rebuilding) + _frame_depth.reset(token) @classmethod def _lookup_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'tuple[Optional[Type[ProtocolBase]], Optional[str]]': # pylint: disable=line-too-long @@ -2288,7 +2346,8 @@ def _parse_next_layer(klass: 'Type[ProtocolBase]', file_: 'bytes', length: 'int' Returns: Instance of ``klass``, or of :class:`~pcapkit.protocols.misc.raw.Raw` - when fewer octets were captured than its header needs. + when fewer octets were captured than its header needs, or when it is + past :data:`FRAME_LAYER_LIMIT`. Note: :meth:`Schema.unpack ` @@ -2317,14 +2376,44 @@ def _parse_next_layer(klass: 'Type[ProtocolBase]', file_: 'bytes', length: 'int' still warns. A layer captured at least as long as its hint cannot be replaced, and is parsed holding nothing, even inside one that holds. + A layer past :data:`FRAME_LAYER_LIMIT` in its frame is not parsed: + it and the rest of the frame are kept as + :class:`~pcapkit.protocols.misc.raw.Raw`, marked + :attr:`_past_layer_limit`, with one + :exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1610`). + + Warns: + ProtocolWarning: If the next layer is past :data:`FRAME_LAYER_LIMIT`. + """ + depth = _frame_depth.get() + 1 + if depth > FRAME_LAYER_LIMIT: + from pcapkit.protocols.misc.null import NoPayload # isort: skip # pylint: disable=import-outside-toplevel + from pcapkit.protocols.misc.raw import Raw # isort: skip # pylint: disable=import-outside-toplevel + + # NOTE: Nothing is lost if the next layer is raw already, or empty. + if klass is not Raw and klass is not NoPayload: + message = (f'the frame reached the limit of {FRAME_LAYER_LIMIT} layers; the rest of it, ' + f'from {klass.__name__} on, is left undissected') + # NOTE: Inside a payload being rebuilt, the layer it is rebuilt + # for warns of the same octets once its own parse gets here. + if not _rebuilding.get(): + warn(message, ProtocolWarning, stacklevel=stacklevel()) + raw = Raw(file_, length, **{**kwargs, 'error': message}) + raw._past_layer_limit = True # pylint: disable=protected-access + return raw + captured = min(length, len(file_)) - with hold_warnings(_short_of_hint(klass, captured)) as held: - next_ = klass(file_, length, **kwargs) # type: ignore[abstract] + token = _frame_depth.set(depth) + try: + with hold_warnings(_short_of_hint(klass, captured)) as held: + next_ = klass(file_, length, **kwargs) # type: ignore[abstract] - length_hint = getattr(next_, '__length_hint__', None) - hint = None if length_hint is None else length_hint() - held.discard = hint is not None and captured < hint and captured < next_.length + length_hint = getattr(next_, '__length_hint__', None) + hint = None if length_hint is None else length_hint() + held.discard = hint is not None and captured < hint and captured < next_.length + finally: + _frame_depth.reset(token) if held.discard: from pcapkit.protocols.misc.raw import Raw # isort: skip # pylint: disable=import-outside-toplevel next_ = Raw(file_, length, **kwargs) diff --git a/pcapkit/toolkit/pcap.py b/pcapkit/toolkit/pcap.py index 377fbf03be..5ddb004091 100644 --- a/pcapkit/toolkit/pcap.py +++ b/pcapkit/toolkit/pcap.py @@ -118,7 +118,9 @@ def tcp_segment(frame: 'ProtocolBase') -> 'Optional[TCPSegment]': (:rfc:`9293#section-3.1`) if the payload of the innermost IP layer is a :class:`~pcapkit.protocols.misc.raw.Raw` the TCP parser rejected, of protocol TCP, from a datagram that is not a later fragment, with at - least those 20 octets captured; else :data:`None`. + least those 20 octets captured; else :data:`None`. A segment past + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` is kept raw + without being parsed, not rejected, so it is not read (:issue:`1610`). Note: A capture snapped inside the TCP header leaves a Data Offset that runs @@ -154,7 +156,10 @@ def tcp_segment(frame: 'ProtocolBase') -> 'Optional[TCPSegment]': raw = ip.payload # NOTE: a Raw with no error is one the caller asked for, by stopping the # dissection at a ``layer`` or ``protocol``, not a segment TCP rejected. + # Nor is one past FRAME_LAYER_LIMIT, which carries an error all the same: + # the frame is not dissected past it, so its TCP is not read (#1610). if (not isinstance(raw, Raw) or raw.info.error is None + or raw._past_layer_limit # pylint: disable=protected-access or ip_info.protocol != Enum_TransType.TCP): return None diff --git a/tests/protocols/test_frame_layer_limit_1610_unit.py b/tests/protocols/test_frame_layer_limit_1610_unit.py new file mode 100644 index 0000000000..af403f7be0 --- /dev/null +++ b/tests/protocols/test_frame_layer_limit_1610_unit.py @@ -0,0 +1,397 @@ +# -*- coding: utf-8 -*- +"""A frame is dissected to 64 layers, and no further. + +GitHub issue #1610: each layer dissects the next one from inside its own parse, +so deep nesting of any protocol ran into Python's recursion limit, and +:func:`~pcapkit.utilities.decorators.beholder` caught the ``RecursionError`` and +left the rest raw, silently. From a script on Python 3.14, TCP was lost behind +108 IPv4 headers in IPv4, or behind 200 stacked 802.1Q tags. #1604 bounded one +chain of extension headers, but its count restarts at every tunnel. + +Now every frame stops at +:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers, counted along its +protocol chain from the first layer its record carries, through tunnels, tags +and extension headers alike: what follows is kept as +:class:`~pcapkit.protocols.misc.raw.Raw`, with one +:exc:`~pcapkit.utilities.warnings.ProtocolWarning`. + +Every case builds its own octets in memory and reads no capture. Classes are +imported inside each test, after :func:`~tests._support.reimport_once_per_class`. + +""" + +import io +import itertools +import logging +import struct +import sys +import time +import unittest +import warnings +from typing import TYPE_CHECKING + +from tests._support import close_extractor, reimport_once_per_class, scale_timeout, time_limit +from tests.foundation import _roundtrip as wire + +if TYPE_CHECKING: + from typing import Optional + +#: Little-endian PCAP global header: v2.4, snaplen 65535, Ethernet. +GLOBAL = bytes.fromhex('d4c3b2a1' '0200' '0400' '00000000' '00000000' 'ffff0000' '01000000') + +#: EtherType naming each layer, from Ethernet or an 802.1Q tag. +ETHERTYPE = {'ipv4': 0x0800, 'ipv6': 0x86DD, 'vlan': 0x8100} +#: Protocol number naming each layer, from IP or an extension header. +PROTO = {'ipv4': 4, 'ipv6': 41, 'opts': 60, 'tcp': 6} + +#: Frames longer than the limit, in layers: by one, and by far. +PAST = (65, 1000) + +#: Layer by layer, after Ethernet, and each ending in TCP: ``count`` layers in +#: all, Ethernet the first and TCP the last. +SHAPES = { + 'IPv4 in IPv4': lambda count: ['ipv4'] * (count - 2) + ['tcp'], + 'IPv6 in IPv6': lambda count: ['ipv6'] * (count - 2) + ['tcp'], + '802.1Q tags': lambda count: ['vlan'] * (count - 3) + ['ipv4', 'tcp'], + # Tags, then IPv4 and IPv6 in turn, with Destination Options between them: + # dissected one by one after IPv4, walked by IPv6. + 'mixed': lambda count: (['vlan'] * 3 + list(itertools.islice(itertools.cycle( + ['ipv4', 'opts', 'opts', 'ipv6', 'opts', 'opts', 'opts']), count - 5)) + ['tcp']), +} + + +#: A TCP header whose Data Offset claims 32 octets, all of them present: snap +#: four off and the parser rejects it, leaving it to the #1518 fall-back. +SNAPPABLE = (struct.pack('!HHIIBBHHH', 40000, 9, 1, 0, 8 << 4, 0x10, 65535, 0, 0) + + bytes.fromhex('020405b4' '01030306' '01010402')) + + +def octets(layers: 'list[str]', sport: int = 40000, segment: 'Optional[bytes]' = None) -> bytes: + """The octets of ``layers``, outermost first, the last of them TCP from + ``sport``, or ``segment`` if given.""" + data, inner = b'', None + for name in reversed(layers): + if name == 'tcp': + data = wire.tcp(b'payload', seq=1, sport=sport) if segment is None else segment + elif name == 'ipv4': + data = wire.ipv4(data, proto=PROTO[inner]) + elif name == 'ipv6': + data = wire.ipv6(data, nxt=PROTO[inner]) + elif name == 'opts': # Hdr Ext Len 0, one PadN + data = bytes([PROTO[inner], 0, 1, 4, 0, 0, 0, 0]) + data + else: # an 802.1Q tag: TCI 0, then the inner EtherType + data = bytes(2) + ETHERTYPE[inner].to_bytes(2, 'big') + data + inner = name + return data + + +def ethernet(layers: 'list[str]', sport: int = 40000, segment: 'Optional[bytes]' = None) -> bytes: + """An Ethernet frame carrying ``layers``.""" + return wire.ethernet(octets(layers, sport, segment), ETHERTYPE[layers[0]]) + + +def record(data: bytes, snapped: int = 0) -> bytes: + """A PCAP record of ``data``, with its last ``snapped`` octets left uncaptured.""" + captured = data[:len(data) - snapped] + return struct.pack(' 'list[str]': + """``times`` tunnels of ``carrier``, each carrying ``count`` Destination Options headers.""" + return ([carrier] + ['opts'] * count) * times + ['tcp'] + + +class _Logged(logging.Handler): + """Collect what the ``pcapkit`` logger reports of a recursion error.""" + + def __init__(self) -> None: + super().__init__(logging.DEBUG) + self.messages = [] # type: list[str] + + def emit(self, entry: 'logging.LogRecord') -> None: + message = entry.getMessage() + if 'recursion' in message.lower(): + self.messages.append(message) + + def __enter__(self) -> '_Logged': + logging.getLogger('pcapkit').addHandler(self) + return self + + def __exit__(self, *exc: object) -> None: + logging.getLogger('pcapkit').removeHandler(self) + + +def depth(func: 'object') -> int: + """How many Python frames deeper than its caller ``func()`` goes.""" + state = {'now': 0, 'most': 0} + + def profile(frame: object, event: str, arg: object) -> None: # pylint: disable=unused-argument + if event == 'call': + state['now'] += 1 + state['most'] = max(state['most'], state['now']) + elif event == 'return': + state['now'] -= 1 + + sys.setprofile(profile) + try: + func() # type: ignore[operator] + finally: + sys.setprofile(None) + return state['most'] + + +class FrameLayerLimitTests(unittest.TestCase): + """C.f. #1610.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def frame(self, data: bytes, snapped: int = 0, **kwargs: object) -> 'tuple[object, list[str]]': + """The PCAP frame of ``data``, and the messages of the ProtocolWarnings it gave.""" + from pcapkit.protocols.misc.pcap.frame import Frame + from pcapkit.protocols.misc.pcap.header import Header + from pcapkit.utilities.warnings import ProtocolWarning + + header = Header(GLOBAL).info + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + parsed = Frame(io.BytesIO(record(data, snapped)), num=1, header=header, **kwargs) + return parsed, [str(item.message) for item in caught if issubclass(item.category, ProtocolWarning)] + + def raw_of(self, parsed: object) -> object: + """The first :class:`~pcapkit.protocols.misc.raw.Raw` layer of ``parsed``.""" + from pcapkit.protocols.misc.raw import Raw + + layer = parsed + while not isinstance(layer, Raw): + layer = layer.payload # type: ignore[attr-defined] + return layer + + def assert_cut(self, layers: 'list[str]', parsed: object, messages: 'list[str]') -> None: + """``parsed`` is the frame of ``layers``, cut at the 65th, with one warning.""" + self.assertNotIn('TCP', parsed) + self.assertEqual(len(parsed.protochain), 65) # type: ignore[attr-defined] + self.assertEqual(len(messages), 1, messages) + self.assertIn('limit of 64 layers', messages[0]) + + # the 65th layer is the 64th after Ethernet, and the rest is kept as captured + layer = self.raw_of(parsed) + self.assertEqual(bytes(layer.data), octets(layers[63:])) # type: ignore[attr-defined] + self.assertEqual(layer.info.error, messages[0]) # type: ignore[attr-defined] + named = ETHERTYPE if layers[62] == 'vlan' else PROTO + self.assertEqual(int(layer.info.protocol), named[layers[63]]) # type: ignore[attr-defined] + + def test_the_limit_is_64(self) -> None: + from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT + + self.assertEqual(FRAME_LAYER_LIMIT, 64) + + def test_a_frame_of_64_layers_is_dissected_to_the_tcp(self) -> None: + # Guard: below the old recursion cut-off, so this held before #1610 too. + for shape, build in SHAPES.items(): + with self.subTest(shape=shape): + parsed, messages = self.frame(ethernet(build(64))) + self.assertIn('TCP', parsed) + # the 64 layers, and the TCP payload, left raw as before + self.assertEqual(len(parsed.protochain), 65) # type: ignore[attr-defined] + self.assertEqual(messages, []) + + def test_a_frame_past_64_layers_stops_at_the_limit(self) -> None: + for shape, build in SHAPES.items(): + for count in PAST: + with self.subTest(shape=shape, count=count): + layers = build(count) + parsed, messages = self.frame(ethernet(layers)) + self.assert_cut(layers, parsed, messages) + + def test_tunnels_between_extension_header_chains_do_not_restart_the_count(self) -> None: + # Three chains of 30, each under the extension header limit of 32, and + # each in a tunnel of its own: the third chain's first header is the 65th + # layer. + for carrier in ('ipv4', 'ipv6'): + with self.subTest(carrier=carrier): + layers = chains(carrier, 30, 3) + parsed, messages = self.frame(ethernet(layers)) + self.assert_cut(layers, parsed, messages) + names = str(parsed.protochain).split(':') # type: ignore[attr-defined] + self.assertEqual(names.count('IPv6-Opts'), 60) + + def test_what_a_walked_header_dissects_counts_after_every_header_walked(self) -> None: + # IPv6 walks its extension headers in a loop, but ESP dissects its + # decrypted payload itself, walked or not. Behind 30 headers walked, + # ESP is the 33rd layer, so a TCP behind 31 IPv4 headers in it is the + # 65th. ESP-NULL decrypts with no keys. + from pcapkit.const.ipv6.extension_header import ExtensionHeader + from pcapkit.corekit.context import ContextRegistry + from pcapkit.protocols.internet.esp import ESPContext, SecurityAssociation + + registry = ContextRegistry.make(ESPContext(SecurityAssociation(spi=7))) + walked = b''.join(bytes([60 if index < 29 else 50, 0, 1, 4, 0, 0, 0, 0]) for index in range(30)) + for nested, cut in ((30, False), (31, True)): + with self.subTest(nested=nested): + inner = octets(['ipv4'] * nested + ['tcp']) + pad = -(len(inner) + 2) % 4 # ESP-NULL: padding, its length, Next Header 4 + esp = struct.pack('!II', 7, 1) + inner + bytes(range(1, pad + 1)) + bytes([pad, 4]) + parsed, messages = self.frame(wire.ethernet(wire.ipv6(walked + esp, nxt=60), 0x86DD), + __context__=registry) + + # walked as an extension header, ESP refuses ``payload`` and + # ``protochain``, so what it dissected is read off ``_next`` + names, layer = [], parsed.payload.payload.extension_headers[ExtensionHeader.ESP] # type: ignore[attr-defined] + while layer._next.__class__.__name__ not in ('Raw', 'NoPayload'): # pylint: disable=protected-access + layer = layer._next # pylint: disable=protected-access + names.append(layer.__class__.__name__) + self.assertEqual(names, ['IPv4'] * nested + ([] if cut else ['TCP'])) + self.assertEqual(layer._next._past_layer_limit, cut) # pylint: disable=protected-access + self.assertEqual(len(messages), int(cut), messages) + + def test_where_both_limits_fall_on_one_layer_it_warns_once(self) -> None: + # Guard: the 33rd header in a row is also the 65th layer, and the + # extension header limit, which held before #1610 too, is the one named. + for carrier in ('ipv4', 'ipv6'): + with self.subTest(carrier=carrier): + layers = ['ipv4'] * 30 + [carrier] + ['opts'] * 40 + ['tcp'] + parsed, messages = self.frame(ethernet(layers)) + self.assertNotIn('TCP', parsed) + self.assertEqual(len(messages), 1, messages) + self.assertIn('limit of 32 headers', messages[0]) + self.assertEqual(len(parsed.protochain), 65) # type: ignore[attr-defined] + + def test_the_count_is_per_frame(self) -> None: + from pcapkit import extract + + # one TCP flow per frame, and so one datagram per frame dissected to its TCP + lengths = (64, 65, 64, 1000, 64) + data = GLOBAL + b''.join(record(ethernet(SHAPES['IPv4 in IPv4'](count), 40000 + number)) + for number, count in enumerate(lengths)) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + extractor = extract(fin=io.BytesIO(data), nofile=True, engine='default', + reassembly=True, tcp=True) + close_extractor(extractor) + self.assertEqual(['TCP' in frame for frame in extractor.frame], [True, False, True, False, True]) + self.assertEqual(sorted(datagram.index[0] for datagram in extractor.reassembly.tcp), [1, 3, 5]) + + def test_a_tcp_segment_past_the_limit_is_not_read_back_out_of_the_raw(self) -> None: + # The layer past the limit is kept raw with an ``error``, as a TCP header + # the parser rejected is, and in every shape here the IP layer above it + # carries TCP. ``tcp_segment`` reads the fixed header of a rejected one + # (#1518), and read this one too, so the TCP 65th layer was reassembled. + from pcapkit.toolkit.pcap import tcp_segment + + for shape, build in SHAPES.items(): + with self.subTest(shape=shape): + parsed, _ = self.frame(ethernet(build(65))) + self.assertIsNone(tcp_segment(parsed)) + whole, _ = self.frame(ethernet(build(64))) + self.assertIsNotNone(tcp_segment(whole)) + + # A header the parser rejected is still read below the limit, but not + # past it. + for count, read in ((3, True), (64, True), (65, False)): + with self.subTest(rejected=count): + parsed, _ = self.frame(ethernet(SHAPES['IPv4 in IPv4'](count), segment=SNAPPABLE), snapped=4) + self.assertNotIn('TCP', parsed) + segment = tcp_segment(parsed) + self.assertEqual(segment is not None, read) + if read: + self.assertEqual((segment.srcport, segment.dstport, len(segment.header)), (40000, 9, 28)) + + def test_a_protocol_constructed_directly_counts_from_the_layer_below_it(self) -> None: + from pcapkit.protocols.link.ethernet import Ethernet + + layers = SHAPES['IPv4 in IPv4'](66) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + whole = Ethernet(ethernet(layers[1:])) + cut = Ethernet(ethernet(layers)) + self.assertIn('TCP', whole) + self.assertNotIn('TCP', cut) + self.assertEqual(len(caught), 1, [str(item.message) for item in caught]) + self.assertEqual(len(cut.protochain), 66) + + def test_no_recursion_error_at_any_depth_and_the_time_stays_bounded(self) -> None: + from pcapkit.protocols.link.ethernet import Ethernet + + def best(data: bytes) -> float: + times = [] + for _ in range(3): + start = time.perf_counter() + Ethernet(data) + times.append(time.perf_counter() - start) + return min(times) + + cases = {'IPv4 in IPv4': 3000, '802.1Q tags': 10000, 'IPv6 in IPv6': 1500, 'mixed': 3000} + with time_limit(scale_timeout(60)), warnings.catch_warnings(), _Logged() as logged: + warnings.simplefilter('ignore') + for shape, count in cases.items(): + with self.subTest(shape=shape): + build = SHAPES[shape] + short, deep = best(ethernet(build(66))), best(ethernet(build(count))) + self.assertLess(deep, 4 * short + 0.05, (short, deep)) + self.assertEqual(logged.messages, []) + + def test_a_cut_frame_rebuilds_byte_exact(self) -> None: + from pcapkit.protocols.link.ethernet import Ethernet + from pcapkit.protocols.misc.pcap.frame import Frame + from pcapkit.protocols.misc.pcap.header import Header + + header = Header(GLOBAL).info + shapes = dict(SHAPES, chains=lambda count: chains('ipv4', 30, count // 31 + 1)) + with warnings.catch_warnings(), _Logged() as logged: + warnings.simplefilter('ignore') + for shape, build in shapes.items(): + for count in PAST: + with self.subTest(shape=shape, count=count): + data = ethernet(build(count)) + parsed, _ = self.frame(data) + self.assertEqual(parsed.data, record(data)) # type: ignore[attr-defined] + rebuilt = Frame.from_data(parsed.info.to_dict(), num=1, header=header) # type: ignore[attr-defined] + self.assertEqual(rebuilt.data, record(data)) + self.assertEqual(str(rebuilt.protochain), str(parsed.protochain)) # type: ignore[attr-defined] + + packet = Ethernet(data) + self.assertEqual(Ethernet.from_data(packet.info.to_dict()).data, data) + self.assertEqual(logged.messages, []) + + def test_rebuilding_a_cut_frame_warns_once_as_parsing_it_does(self) -> None: + # Each layer rebuilt parses its own octets again, and each such parse + # reaches the limit, so the rebuild warned once a layer: 65 times. + from pcapkit.protocols.misc.pcap.frame import Frame + from pcapkit.protocols.misc.pcap.header import Header + from pcapkit.utilities.warnings import ProtocolWarning + + header = Header(GLOBAL).info + for shape, build in SHAPES.items(): + for count in PAST: + with self.subTest(shape=shape, count=count): + parsed, messages = self.frame(ethernet(build(count))) + self.assertEqual(len(messages), 1, messages) + for data in (parsed.info, parsed.info.to_dict()): # type: ignore[attr-defined] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + Frame.from_data(data, num=1, header=header) + self.assertEqual([str(item.message) for item in caught + if issubclass(item.category, ProtocolWarning)], messages) + + def test_rebuilding_a_cut_frame_goes_no_deeper_than_parsing_it(self) -> None: + # Each layer rebuilt parses its own octets again, so were it to count + # from itself, the rebuild would go 64 layers below every layer of the + # frame, nearly twice as deep as the parse, and past the recursion limit. + # It may go a frame or two deeper than the parse, from ``from_data`` + # itself, but not a layer's worth. + from pcapkit.protocols.link.ethernet import Ethernet + + for carrier in ('ipv4', 'ipv6'): + with self.subTest(carrier=carrier), warnings.catch_warnings(), _Logged() as logged: + warnings.simplefilter('ignore') + data = ethernet(chains(carrier, 30, 33) if carrier == 'ipv4' else SHAPES['IPv6 in IPv6'](1000)) + info = Ethernet(data).info.to_dict() + parsing = depth(lambda: Ethernet(data)) # pylint: disable=cell-var-from-loop + rebuilding = depth(lambda: Ethernet.from_data(info)) # pylint: disable=cell-var-from-loop + self.assertLess(rebuilding - parsing, 9, (parsing, rebuilding)) + self.assertEqual(logged.messages, []) + + +if __name__ == '__main__': + unittest.main()