diff --git a/docs/source/contributing/pep.rst b/docs/source/contributing/pep.rst index 1e638ccef3..5406e2c4ba 100644 --- a/docs/source/contributing/pep.rst +++ b/docs/source/contributing/pep.rst @@ -417,6 +417,33 @@ Two traps for anyone benchmarking this library. ``reassembly=True`` and only the switch measures nothing; and timing several capture shapes in one process inflates them by up to 73%, so each shape wants its own interpreter. +Recursive Dissection, With a Depth Limit +---------------------------------------- + +**Decided for now, open to revisit.** Each layer dissects the next from inside +its own parse, so dissecting a frame recurses once per layer, at 9 to 12 Python +frames a layer. Deep nesting -- IP in IP, stacked 802.1Q tags, long chains of +IPv6 extension headers -- used to exhaust the recursion limit and leave the rest +of the frame raw, silently, at a depth that varied with the interpreter and the +caller's stack (:issue:`1604`, :issue:`1610`). + +Recursion stays, with a counted bound: +:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` caps a frame at 64 layers, +and :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` caps one +chain of extension headers at 32. Past either, the rest stays +:class:`~pcapkit.protocols.misc.raw.Raw`, with one +:exc:`~pcapkit.utilities.warnings.ProtocolWarning`. Raising +:func:`sys.setrecursionlimit` instead was rejected: it is process-wide, and the +cut-off would still move with the caller's stack. The bound fits with room to +spare, but not unlimited room. Under :program:`pytest` at the default limit of +1,000, the costliest 64-layer frame needs a limit of 834 on Python 3.11 and 756 +on 3.14. That frame is IPv4 carrying a chain of 32 extension headers, then IPv4 +in IPv4 carrying 28 more, then TCP. + +An iterative model would need no such bound: a loop that dissects one layer and +takes the next from it. It changes how every protocol hands over its payload, +so it is a design of its own, and may replace the recursion in a future release. + Logging Integration ------------------- diff --git a/docs/source/pcapkit/protocols/protocol.rst b/docs/source/pcapkit/protocols/protocol.rst index b6dc2dab1a..43a96eba65 100644 --- a/docs/source/pcapkit/protocols/protocol.rst +++ b/docs/source/pcapkit/protocols/protocol.rst @@ -73,6 +73,7 @@ Internal Definitions below. .. automethod:: _lookup_next_layer .. automethod:: _decode_next_layer .. automethod:: _import_next_layer + .. automethod:: _parse_next_layer .. automethod:: _get_context .. autoattribute:: _data @@ -82,6 +83,7 @@ Internal Definitions below. .. autoattribute:: _protos .. autoattribute:: _seekset .. autoattribute:: _sigterm + .. autoattribute:: _past_layer_limit .. autoattribute:: __data__ .. automethod:: __init__ @@ -99,6 +101,26 @@ Internal Definitions below. .. autoattribute:: _exproto .. autoattribute:: _exctx +Layers per Frame +---------------- + +A frame is dissected to at most +:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of its protocol +chain, counted from the first layer the frame record carries, through tunnels, +VLAN tags and IPv6 extension headers alike. What follows is kept as +:class:`~pcapkit.protocols.misc.raw.Raw`, with one +:exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1610`), and is +marked :attr:`~pcapkit.protocols.protocol.Protocol._past_layer_limit`, so the +default engine's TCP reassembly and flow tracing do not read a TCP segment +there back out, as they do one the TCP parser rejected (:issue:`1518`). A +protocol constructed from its own octets is not counted itself, so the count +starts at the first layer it dissects. One chain of extension headers is also bounded on +its own, by :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT`. +The ``dpkt``, ``scapy``, ``pyshark`` and ``pypcapfile`` engines dissect with +their own parsers, which do not apply this bound (:issue:`1609`). + +.. autodata:: pcapkit.protocols.protocol.FRAME_LAYER_LIMIT + Data Models ----------- diff --git a/pcapkit/protocols/internet/ipv6.py b/pcapkit/protocols/internet/ipv6.py index 16cb16bd05..66b163dfe1 100644 --- a/pcapkit/protocols/internet/ipv6.py +++ b/pcapkit/protocols/internet/ipv6.py @@ -37,6 +37,7 @@ from pcapkit.protocols.data.internet.ipv6 import IPv6 as Data_IPv6 from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT from pcapkit.protocols.internet.ip import IP +from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT, _frame_depth from pcapkit.protocols.schema.internet.ipv6 import IPv6 as Schema_IPv6 from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length from pcapkit.utilities.decorators import beholder @@ -535,6 +536,7 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, _exthdr = [] # (parser class, info) per extension header split = None # type: Optional[tuple[int, int]] # at first Fragment header opaque = False # walk ended at a non-zero Fragment Offset + depth = _frame_depth.get() # layers of the frame up to this one # traverse if next header is an extension header payload = self.__header__.get_payload() @@ -553,14 +555,27 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, if len(_protos) >= EXTENSION_HEADER_LIMIT: break + # NOTE: Likewise past :data:`FRAME_LAYER_LIMIT + # ` layers of the + # frame (:issue:`1610`), since each header walked is one of them. + if depth + len(_protos) >= FRAME_LAYER_LIMIT: + break + # # directly break when No Next Header occurs # if proto.name == 'IPv6-NoNxt': # proto = None # break # make protocol name - next_ = self._import_next_layer(proto, packet=packet, version=6, extension=True, - payload=payload) # type: ignore[misc,call-arg,arg-type] + # NOTE: Counted after every header walked, as it would be were each + # dissected by the one before it. ESP dissects its decrypted + # payload even when walked, and those layers count on from here. + token = _frame_depth.set(depth + len(_protos)) + try: + next_ = self._import_next_layer(proto, packet=packet, version=6, extension=True, + payload=payload) # type: ignore[misc,call-arg,arg-type] + finally: + _frame_depth.reset(token) info = next_.info # A layer with no ``next`` field cannot safely continue the walk. @@ -657,8 +672,12 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None, # ``fragment`` above, which this leaves as it was. ipv6_exthdr = ProtoChain.from_list(_protos) # type: ignore[arg-type] self._exthdr_depth = len(_protos) # the headers the upper layer follows - return super()._decode_next_layer(ipv6, None if opaque else proto, raw_len, packet=packet, - ipv6_exthdr=ipv6_exthdr, payload=payload) + token = _frame_depth.set(depth + len(_protos)) + try: + return super()._decode_next_layer(ipv6, None if opaque else proto, raw_len, packet=packet, + ipv6_exthdr=ipv6_exthdr, payload=payload) + finally: + _frame_depth.reset(token) @beholder # type: ignore[arg-type] def _import_next_layer(self, proto: 'int', length: 'Optional[int]' = None, *, # pylint: disable=arguments-differ diff --git a/pcapkit/protocols/protocol.py b/pcapkit/protocols/protocol.py index db087f35a5..a600d6a5bb 100644 --- a/pcapkit/protocols/protocol.py +++ b/pcapkit/protocols/protocol.py @@ -16,6 +16,7 @@ import collections import collections.abc import contextlib +import contextvars import difflib import enum import functools @@ -49,8 +50,9 @@ from pcapkit.utilities.decorators import beholder, seekset from pcapkit.utilities.exceptions import (BaseError, ProtocolError, ProtocolNotFound, ProtocolNotImplemented, RegistryError, StructError, - UnsupportedCall) -from pcapkit.utilities.warnings import RegistryWarning, UnknownFieldWarning, hold_warnings, warn + UnsupportedCall, stacklevel) +from pcapkit.utilities.warnings import (ProtocolWarning, RegistryWarning, UnknownFieldWarning, + hold_warnings, warn) if TYPE_CHECKING: from enum import IntEnum as StdlibEnum @@ -389,6 +391,37 @@ def _check_construction_keywords(cls: 'type', kwargs: 'dict[str, Any]', UnknownFieldWarning) +#: Most layers dissected in one frame (:issue:`1610`). A layer is one entry of +#: the frame's protocol chain, counted from the first layer the frame record +#: carries, usually its link layer; a protocol constructed from its own octets +#: is not counted itself, so the count starts at the first layer it dissects. +#: IPv6 extension headers are layers like any other, whether +#: :class:`~pcapkit.protocols.internet.ipv6.IPv6` walks them or each dissects +#: the next. What follows the last layer is kept as +#: :class:`~pcapkit.protocols.misc.raw.Raw`, with one +#: :exc:`~pcapkit.utilities.warnings.ProtocolWarning`, unless it would have been +#: left raw anyway or there is nothing left. +#: +#: Each layer dissects the next from inside its own parse, so without the bound +#: a deep enough frame, such as IP in IP or stacked 802.1Q tags, stops wherever +#: Python's recursion limit falls, which depends on the interpreter and on how +#: deep the caller's own stack is. A tunnel does not restart the count, and nor +#: does the end of a chain of extension headers, which +#: :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` bounds on +#: its own. +FRAME_LAYER_LIMIT = 64 + +#: contextvars.ContextVar[int]: How many layers of the frame being dissected +#: come before the next one :meth:`ProtocolBase._parse_next_layer` parses, c.f. +#: :data:`FRAME_LAYER_LIMIT`. +_frame_depth = contextvars.ContextVar('_frame_depth', default=0) + +#: contextvars.ContextVar[bool]: Whether a payload is being rebuilt by +#: :meth:`ProtocolBase._make_payload`, so that the layer it is rebuilt for +#: parses the same octets again, c.f. :data:`FRAME_LAYER_LIMIT`. +_rebuilding = contextvars.ContextVar('_rebuilding', default=False) + + def _short_of_hint(klass: 'Type[ProtocolBase]', captured: 'int') -> 'bool': """Tell whether ``captured`` octets may be too few for a ``klass`` header. @@ -536,6 +569,13 @@ class ProtocolBase(Generic[_PT, _ST], metaclass=ProtocolMeta): #: ``object.__new__(SomeProtocol)``) sees, so reading it is always safe. _exctx: 'Optional[ContextRegistry]' = None + #: Whether this layer is past :data:`FRAME_LAYER_LIMIT`, and so was kept as + #: :class:`~pcapkit.protocols.misc.raw.Raw` by :meth:`_parse_next_layer` + #: without being parsed. Such a layer carries an ``error`` as one that + #: failed to parse does, so this is what tells the two apart: what follows + #: the limit is not to be read on (:issue:`1610`). + _past_layer_limit: 'bool' = False + ########################################################################## # Properties. ########################################################################## @@ -2017,6 +2057,14 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc neither. The payload is then found from the structure of ``data`` instead, by :meth:`_lookup_payload`. + The payload is built one layer further into the frame than this + protocol, as :meth:`_parse_next_layer` counts it, since each layer built + parses its own octets again. So rebuilding a frame stops at + :data:`FRAME_LAYER_LIMIT` where parsing it did, rather than that many + layers below each layer of it (:issue:`1610`). Nor does the payload's + parse warn of the limit: this protocol's own parse goes over the same + octets, so the rebuilt frame warns of it once, as the parsed one did. + Args: data: protocol data exclude: keys of ``data`` that are not the payload, though they may @@ -2032,7 +2080,12 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc from pcapkit.protocols.misc.null import \ NoPayload # pylint: disable=import-outside-toplevel return NoPayload() - return proto.from_data(data[name]) + token, rebuilding = _frame_depth.set(_frame_depth.get() + 1), _rebuilding.set(True) + try: + return proto.from_data(data[name]) + finally: + _rebuilding.reset(rebuilding) + _frame_depth.reset(token) proto = cast('Optional[Type[Protocol]]', data.get('__next_type__')) if proto is None or not (isinstance(proto, type) and issubclass(proto, ProtocolBase)): @@ -2046,7 +2099,12 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc NoPayload # pylint: disable=import-outside-toplevel return NoPayload() - return proto.from_data(data[name]) + token, rebuilding = _frame_depth.set(_frame_depth.get() + 1), _rebuilding.set(True) + try: + return proto.from_data(data[name]) + finally: + _rebuilding.reset(rebuilding) + _frame_depth.reset(token) @classmethod def _lookup_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'tuple[Optional[Type[ProtocolBase]], Optional[str]]': # pylint: disable=line-too-long @@ -2288,7 +2346,8 @@ def _parse_next_layer(klass: 'Type[ProtocolBase]', file_: 'bytes', length: 'int' Returns: Instance of ``klass``, or of :class:`~pcapkit.protocols.misc.raw.Raw` - when fewer octets were captured than its header needs. + when fewer octets were captured than its header needs, or when it is + past :data:`FRAME_LAYER_LIMIT`. Note: :meth:`Schema.unpack ` @@ -2317,14 +2376,44 @@ def _parse_next_layer(klass: 'Type[ProtocolBase]', file_: 'bytes', length: 'int' still warns. A layer captured at least as long as its hint cannot be replaced, and is parsed holding nothing, even inside one that holds. + A layer past :data:`FRAME_LAYER_LIMIT` in its frame is not parsed: + it and the rest of the frame are kept as + :class:`~pcapkit.protocols.misc.raw.Raw`, marked + :attr:`_past_layer_limit`, with one + :exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1610`). + + Warns: + ProtocolWarning: If the next layer is past :data:`FRAME_LAYER_LIMIT`. + """ + depth = _frame_depth.get() + 1 + if depth > FRAME_LAYER_LIMIT: + from pcapkit.protocols.misc.null import NoPayload # isort: skip # pylint: disable=import-outside-toplevel + from pcapkit.protocols.misc.raw import Raw # isort: skip # pylint: disable=import-outside-toplevel + + # NOTE: Nothing is lost if the next layer is raw already, or empty. + if klass is not Raw and klass is not NoPayload: + message = (f'the frame reached the limit of {FRAME_LAYER_LIMIT} layers; the rest of it, ' + f'from {klass.__name__} on, is left undissected') + # NOTE: Inside a payload being rebuilt, the layer it is rebuilt + # for warns of the same octets once its own parse gets here. + if not _rebuilding.get(): + warn(message, ProtocolWarning, stacklevel=stacklevel()) + raw = Raw(file_, length, **{**kwargs, 'error': message}) + raw._past_layer_limit = True # pylint: disable=protected-access + return raw + captured = min(length, len(file_)) - with hold_warnings(_short_of_hint(klass, captured)) as held: - next_ = klass(file_, length, **kwargs) # type: ignore[abstract] + token = _frame_depth.set(depth) + try: + with hold_warnings(_short_of_hint(klass, captured)) as held: + next_ = klass(file_, length, **kwargs) # type: ignore[abstract] - length_hint = getattr(next_, '__length_hint__', None) - hint = None if length_hint is None else length_hint() - held.discard = hint is not None and captured < hint and captured < next_.length + length_hint = getattr(next_, '__length_hint__', None) + hint = None if length_hint is None else length_hint() + held.discard = hint is not None and captured < hint and captured < next_.length + finally: + _frame_depth.reset(token) if held.discard: from pcapkit.protocols.misc.raw import Raw # isort: skip # pylint: disable=import-outside-toplevel next_ = Raw(file_, length, **kwargs) diff --git a/pcapkit/toolkit/pcap.py b/pcapkit/toolkit/pcap.py index 377fbf03be..5ddb004091 100644 --- a/pcapkit/toolkit/pcap.py +++ b/pcapkit/toolkit/pcap.py @@ -118,7 +118,9 @@ def tcp_segment(frame: 'ProtocolBase') -> 'Optional[TCPSegment]': (:rfc:`9293#section-3.1`) if the payload of the innermost IP layer is a :class:`~pcapkit.protocols.misc.raw.Raw` the TCP parser rejected, of protocol TCP, from a datagram that is not a later fragment, with at - least those 20 octets captured; else :data:`None`. + least those 20 octets captured; else :data:`None`. A segment past + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` is kept raw + without being parsed, not rejected, so it is not read (:issue:`1610`). Note: A capture snapped inside the TCP header leaves a Data Offset that runs @@ -154,7 +156,10 @@ def tcp_segment(frame: 'ProtocolBase') -> 'Optional[TCPSegment]': raw = ip.payload # NOTE: a Raw with no error is one the caller asked for, by stopping the # dissection at a ``layer`` or ``protocol``, not a segment TCP rejected. + # Nor is one past FRAME_LAYER_LIMIT, which carries an error all the same: + # the frame is not dissected past it, so its TCP is not read (#1610). if (not isinstance(raw, Raw) or raw.info.error is None + or raw._past_layer_limit # pylint: disable=protected-access or ip_info.protocol != Enum_TransType.TCP): return None diff --git a/tests/protocols/test_frame_layer_limit_1610_unit.py b/tests/protocols/test_frame_layer_limit_1610_unit.py new file mode 100644 index 0000000000..af403f7be0 --- /dev/null +++ b/tests/protocols/test_frame_layer_limit_1610_unit.py @@ -0,0 +1,397 @@ +# -*- coding: utf-8 -*- +"""A frame is dissected to 64 layers, and no further. + +GitHub issue #1610: each layer dissects the next one from inside its own parse, +so deep nesting of any protocol ran into Python's recursion limit, and +:func:`~pcapkit.utilities.decorators.beholder` caught the ``RecursionError`` and +left the rest raw, silently. From a script on Python 3.14, TCP was lost behind +108 IPv4 headers in IPv4, or behind 200 stacked 802.1Q tags. #1604 bounded one +chain of extension headers, but its count restarts at every tunnel. + +Now every frame stops at +:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers, counted along its +protocol chain from the first layer its record carries, through tunnels, tags +and extension headers alike: what follows is kept as +:class:`~pcapkit.protocols.misc.raw.Raw`, with one +:exc:`~pcapkit.utilities.warnings.ProtocolWarning`. + +Every case builds its own octets in memory and reads no capture. Classes are +imported inside each test, after :func:`~tests._support.reimport_once_per_class`. + +""" + +import io +import itertools +import logging +import struct +import sys +import time +import unittest +import warnings +from typing import TYPE_CHECKING + +from tests._support import close_extractor, reimport_once_per_class, scale_timeout, time_limit +from tests.foundation import _roundtrip as wire + +if TYPE_CHECKING: + from typing import Optional + +#: Little-endian PCAP global header: v2.4, snaplen 65535, Ethernet. +GLOBAL = bytes.fromhex('d4c3b2a1' '0200' '0400' '00000000' '00000000' 'ffff0000' '01000000') + +#: EtherType naming each layer, from Ethernet or an 802.1Q tag. +ETHERTYPE = {'ipv4': 0x0800, 'ipv6': 0x86DD, 'vlan': 0x8100} +#: Protocol number naming each layer, from IP or an extension header. +PROTO = {'ipv4': 4, 'ipv6': 41, 'opts': 60, 'tcp': 6} + +#: Frames longer than the limit, in layers: by one, and by far. +PAST = (65, 1000) + +#: Layer by layer, after Ethernet, and each ending in TCP: ``count`` layers in +#: all, Ethernet the first and TCP the last. +SHAPES = { + 'IPv4 in IPv4': lambda count: ['ipv4'] * (count - 2) + ['tcp'], + 'IPv6 in IPv6': lambda count: ['ipv6'] * (count - 2) + ['tcp'], + '802.1Q tags': lambda count: ['vlan'] * (count - 3) + ['ipv4', 'tcp'], + # Tags, then IPv4 and IPv6 in turn, with Destination Options between them: + # dissected one by one after IPv4, walked by IPv6. + 'mixed': lambda count: (['vlan'] * 3 + list(itertools.islice(itertools.cycle( + ['ipv4', 'opts', 'opts', 'ipv6', 'opts', 'opts', 'opts']), count - 5)) + ['tcp']), +} + + +#: A TCP header whose Data Offset claims 32 octets, all of them present: snap +#: four off and the parser rejects it, leaving it to the #1518 fall-back. +SNAPPABLE = (struct.pack('!HHIIBBHHH', 40000, 9, 1, 0, 8 << 4, 0x10, 65535, 0, 0) + + bytes.fromhex('020405b4' '01030306' '01010402')) + + +def octets(layers: 'list[str]', sport: int = 40000, segment: 'Optional[bytes]' = None) -> bytes: + """The octets of ``layers``, outermost first, the last of them TCP from + ``sport``, or ``segment`` if given.""" + data, inner = b'', None + for name in reversed(layers): + if name == 'tcp': + data = wire.tcp(b'payload', seq=1, sport=sport) if segment is None else segment + elif name == 'ipv4': + data = wire.ipv4(data, proto=PROTO[inner]) + elif name == 'ipv6': + data = wire.ipv6(data, nxt=PROTO[inner]) + elif name == 'opts': # Hdr Ext Len 0, one PadN + data = bytes([PROTO[inner], 0, 1, 4, 0, 0, 0, 0]) + data + else: # an 802.1Q tag: TCI 0, then the inner EtherType + data = bytes(2) + ETHERTYPE[inner].to_bytes(2, 'big') + data + inner = name + return data + + +def ethernet(layers: 'list[str]', sport: int = 40000, segment: 'Optional[bytes]' = None) -> bytes: + """An Ethernet frame carrying ``layers``.""" + return wire.ethernet(octets(layers, sport, segment), ETHERTYPE[layers[0]]) + + +def record(data: bytes, snapped: int = 0) -> bytes: + """A PCAP record of ``data``, with its last ``snapped`` octets left uncaptured.""" + captured = data[:len(data) - snapped] + return struct.pack(' 'list[str]': + """``times`` tunnels of ``carrier``, each carrying ``count`` Destination Options headers.""" + return ([carrier] + ['opts'] * count) * times + ['tcp'] + + +class _Logged(logging.Handler): + """Collect what the ``pcapkit`` logger reports of a recursion error.""" + + def __init__(self) -> None: + super().__init__(logging.DEBUG) + self.messages = [] # type: list[str] + + def emit(self, entry: 'logging.LogRecord') -> None: + message = entry.getMessage() + if 'recursion' in message.lower(): + self.messages.append(message) + + def __enter__(self) -> '_Logged': + logging.getLogger('pcapkit').addHandler(self) + return self + + def __exit__(self, *exc: object) -> None: + logging.getLogger('pcapkit').removeHandler(self) + + +def depth(func: 'object') -> int: + """How many Python frames deeper than its caller ``func()`` goes.""" + state = {'now': 0, 'most': 0} + + def profile(frame: object, event: str, arg: object) -> None: # pylint: disable=unused-argument + if event == 'call': + state['now'] += 1 + state['most'] = max(state['most'], state['now']) + elif event == 'return': + state['now'] -= 1 + + sys.setprofile(profile) + try: + func() # type: ignore[operator] + finally: + sys.setprofile(None) + return state['most'] + + +class FrameLayerLimitTests(unittest.TestCase): + """C.f. #1610.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def frame(self, data: bytes, snapped: int = 0, **kwargs: object) -> 'tuple[object, list[str]]': + """The PCAP frame of ``data``, and the messages of the ProtocolWarnings it gave.""" + from pcapkit.protocols.misc.pcap.frame import Frame + from pcapkit.protocols.misc.pcap.header import Header + from pcapkit.utilities.warnings import ProtocolWarning + + header = Header(GLOBAL).info + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + parsed = Frame(io.BytesIO(record(data, snapped)), num=1, header=header, **kwargs) + return parsed, [str(item.message) for item in caught if issubclass(item.category, ProtocolWarning)] + + def raw_of(self, parsed: object) -> object: + """The first :class:`~pcapkit.protocols.misc.raw.Raw` layer of ``parsed``.""" + from pcapkit.protocols.misc.raw import Raw + + layer = parsed + while not isinstance(layer, Raw): + layer = layer.payload # type: ignore[attr-defined] + return layer + + def assert_cut(self, layers: 'list[str]', parsed: object, messages: 'list[str]') -> None: + """``parsed`` is the frame of ``layers``, cut at the 65th, with one warning.""" + self.assertNotIn('TCP', parsed) + self.assertEqual(len(parsed.protochain), 65) # type: ignore[attr-defined] + self.assertEqual(len(messages), 1, messages) + self.assertIn('limit of 64 layers', messages[0]) + + # the 65th layer is the 64th after Ethernet, and the rest is kept as captured + layer = self.raw_of(parsed) + self.assertEqual(bytes(layer.data), octets(layers[63:])) # type: ignore[attr-defined] + self.assertEqual(layer.info.error, messages[0]) # type: ignore[attr-defined] + named = ETHERTYPE if layers[62] == 'vlan' else PROTO + self.assertEqual(int(layer.info.protocol), named[layers[63]]) # type: ignore[attr-defined] + + def test_the_limit_is_64(self) -> None: + from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT + + self.assertEqual(FRAME_LAYER_LIMIT, 64) + + def test_a_frame_of_64_layers_is_dissected_to_the_tcp(self) -> None: + # Guard: below the old recursion cut-off, so this held before #1610 too. + for shape, build in SHAPES.items(): + with self.subTest(shape=shape): + parsed, messages = self.frame(ethernet(build(64))) + self.assertIn('TCP', parsed) + # the 64 layers, and the TCP payload, left raw as before + self.assertEqual(len(parsed.protochain), 65) # type: ignore[attr-defined] + self.assertEqual(messages, []) + + def test_a_frame_past_64_layers_stops_at_the_limit(self) -> None: + for shape, build in SHAPES.items(): + for count in PAST: + with self.subTest(shape=shape, count=count): + layers = build(count) + parsed, messages = self.frame(ethernet(layers)) + self.assert_cut(layers, parsed, messages) + + def test_tunnels_between_extension_header_chains_do_not_restart_the_count(self) -> None: + # Three chains of 30, each under the extension header limit of 32, and + # each in a tunnel of its own: the third chain's first header is the 65th + # layer. + for carrier in ('ipv4', 'ipv6'): + with self.subTest(carrier=carrier): + layers = chains(carrier, 30, 3) + parsed, messages = self.frame(ethernet(layers)) + self.assert_cut(layers, parsed, messages) + names = str(parsed.protochain).split(':') # type: ignore[attr-defined] + self.assertEqual(names.count('IPv6-Opts'), 60) + + def test_what_a_walked_header_dissects_counts_after_every_header_walked(self) -> None: + # IPv6 walks its extension headers in a loop, but ESP dissects its + # decrypted payload itself, walked or not. Behind 30 headers walked, + # ESP is the 33rd layer, so a TCP behind 31 IPv4 headers in it is the + # 65th. ESP-NULL decrypts with no keys. + from pcapkit.const.ipv6.extension_header import ExtensionHeader + from pcapkit.corekit.context import ContextRegistry + from pcapkit.protocols.internet.esp import ESPContext, SecurityAssociation + + registry = ContextRegistry.make(ESPContext(SecurityAssociation(spi=7))) + walked = b''.join(bytes([60 if index < 29 else 50, 0, 1, 4, 0, 0, 0, 0]) for index in range(30)) + for nested, cut in ((30, False), (31, True)): + with self.subTest(nested=nested): + inner = octets(['ipv4'] * nested + ['tcp']) + pad = -(len(inner) + 2) % 4 # ESP-NULL: padding, its length, Next Header 4 + esp = struct.pack('!II', 7, 1) + inner + bytes(range(1, pad + 1)) + bytes([pad, 4]) + parsed, messages = self.frame(wire.ethernet(wire.ipv6(walked + esp, nxt=60), 0x86DD), + __context__=registry) + + # walked as an extension header, ESP refuses ``payload`` and + # ``protochain``, so what it dissected is read off ``_next`` + names, layer = [], parsed.payload.payload.extension_headers[ExtensionHeader.ESP] # type: ignore[attr-defined] + while layer._next.__class__.__name__ not in ('Raw', 'NoPayload'): # pylint: disable=protected-access + layer = layer._next # pylint: disable=protected-access + names.append(layer.__class__.__name__) + self.assertEqual(names, ['IPv4'] * nested + ([] if cut else ['TCP'])) + self.assertEqual(layer._next._past_layer_limit, cut) # pylint: disable=protected-access + self.assertEqual(len(messages), int(cut), messages) + + def test_where_both_limits_fall_on_one_layer_it_warns_once(self) -> None: + # Guard: the 33rd header in a row is also the 65th layer, and the + # extension header limit, which held before #1610 too, is the one named. + for carrier in ('ipv4', 'ipv6'): + with self.subTest(carrier=carrier): + layers = ['ipv4'] * 30 + [carrier] + ['opts'] * 40 + ['tcp'] + parsed, messages = self.frame(ethernet(layers)) + self.assertNotIn('TCP', parsed) + self.assertEqual(len(messages), 1, messages) + self.assertIn('limit of 32 headers', messages[0]) + self.assertEqual(len(parsed.protochain), 65) # type: ignore[attr-defined] + + def test_the_count_is_per_frame(self) -> None: + from pcapkit import extract + + # one TCP flow per frame, and so one datagram per frame dissected to its TCP + lengths = (64, 65, 64, 1000, 64) + data = GLOBAL + b''.join(record(ethernet(SHAPES['IPv4 in IPv4'](count), 40000 + number)) + for number, count in enumerate(lengths)) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + extractor = extract(fin=io.BytesIO(data), nofile=True, engine='default', + reassembly=True, tcp=True) + close_extractor(extractor) + self.assertEqual(['TCP' in frame for frame in extractor.frame], [True, False, True, False, True]) + self.assertEqual(sorted(datagram.index[0] for datagram in extractor.reassembly.tcp), [1, 3, 5]) + + def test_a_tcp_segment_past_the_limit_is_not_read_back_out_of_the_raw(self) -> None: + # The layer past the limit is kept raw with an ``error``, as a TCP header + # the parser rejected is, and in every shape here the IP layer above it + # carries TCP. ``tcp_segment`` reads the fixed header of a rejected one + # (#1518), and read this one too, so the TCP 65th layer was reassembled. + from pcapkit.toolkit.pcap import tcp_segment + + for shape, build in SHAPES.items(): + with self.subTest(shape=shape): + parsed, _ = self.frame(ethernet(build(65))) + self.assertIsNone(tcp_segment(parsed)) + whole, _ = self.frame(ethernet(build(64))) + self.assertIsNotNone(tcp_segment(whole)) + + # A header the parser rejected is still read below the limit, but not + # past it. + for count, read in ((3, True), (64, True), (65, False)): + with self.subTest(rejected=count): + parsed, _ = self.frame(ethernet(SHAPES['IPv4 in IPv4'](count), segment=SNAPPABLE), snapped=4) + self.assertNotIn('TCP', parsed) + segment = tcp_segment(parsed) + self.assertEqual(segment is not None, read) + if read: + self.assertEqual((segment.srcport, segment.dstport, len(segment.header)), (40000, 9, 28)) + + def test_a_protocol_constructed_directly_counts_from_the_layer_below_it(self) -> None: + from pcapkit.protocols.link.ethernet import Ethernet + + layers = SHAPES['IPv4 in IPv4'](66) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + whole = Ethernet(ethernet(layers[1:])) + cut = Ethernet(ethernet(layers)) + self.assertIn('TCP', whole) + self.assertNotIn('TCP', cut) + self.assertEqual(len(caught), 1, [str(item.message) for item in caught]) + self.assertEqual(len(cut.protochain), 66) + + def test_no_recursion_error_at_any_depth_and_the_time_stays_bounded(self) -> None: + from pcapkit.protocols.link.ethernet import Ethernet + + def best(data: bytes) -> float: + times = [] + for _ in range(3): + start = time.perf_counter() + Ethernet(data) + times.append(time.perf_counter() - start) + return min(times) + + cases = {'IPv4 in IPv4': 3000, '802.1Q tags': 10000, 'IPv6 in IPv6': 1500, 'mixed': 3000} + with time_limit(scale_timeout(60)), warnings.catch_warnings(), _Logged() as logged: + warnings.simplefilter('ignore') + for shape, count in cases.items(): + with self.subTest(shape=shape): + build = SHAPES[shape] + short, deep = best(ethernet(build(66))), best(ethernet(build(count))) + self.assertLess(deep, 4 * short + 0.05, (short, deep)) + self.assertEqual(logged.messages, []) + + def test_a_cut_frame_rebuilds_byte_exact(self) -> None: + from pcapkit.protocols.link.ethernet import Ethernet + from pcapkit.protocols.misc.pcap.frame import Frame + from pcapkit.protocols.misc.pcap.header import Header + + header = Header(GLOBAL).info + shapes = dict(SHAPES, chains=lambda count: chains('ipv4', 30, count // 31 + 1)) + with warnings.catch_warnings(), _Logged() as logged: + warnings.simplefilter('ignore') + for shape, build in shapes.items(): + for count in PAST: + with self.subTest(shape=shape, count=count): + data = ethernet(build(count)) + parsed, _ = self.frame(data) + self.assertEqual(parsed.data, record(data)) # type: ignore[attr-defined] + rebuilt = Frame.from_data(parsed.info.to_dict(), num=1, header=header) # type: ignore[attr-defined] + self.assertEqual(rebuilt.data, record(data)) + self.assertEqual(str(rebuilt.protochain), str(parsed.protochain)) # type: ignore[attr-defined] + + packet = Ethernet(data) + self.assertEqual(Ethernet.from_data(packet.info.to_dict()).data, data) + self.assertEqual(logged.messages, []) + + def test_rebuilding_a_cut_frame_warns_once_as_parsing_it_does(self) -> None: + # Each layer rebuilt parses its own octets again, and each such parse + # reaches the limit, so the rebuild warned once a layer: 65 times. + from pcapkit.protocols.misc.pcap.frame import Frame + from pcapkit.protocols.misc.pcap.header import Header + from pcapkit.utilities.warnings import ProtocolWarning + + header = Header(GLOBAL).info + for shape, build in SHAPES.items(): + for count in PAST: + with self.subTest(shape=shape, count=count): + parsed, messages = self.frame(ethernet(build(count))) + self.assertEqual(len(messages), 1, messages) + for data in (parsed.info, parsed.info.to_dict()): # type: ignore[attr-defined] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + Frame.from_data(data, num=1, header=header) + self.assertEqual([str(item.message) for item in caught + if issubclass(item.category, ProtocolWarning)], messages) + + def test_rebuilding_a_cut_frame_goes_no_deeper_than_parsing_it(self) -> None: + # Each layer rebuilt parses its own octets again, so were it to count + # from itself, the rebuild would go 64 layers below every layer of the + # frame, nearly twice as deep as the parse, and past the recursion limit. + # It may go a frame or two deeper than the parse, from ``from_data`` + # itself, but not a layer's worth. + from pcapkit.protocols.link.ethernet import Ethernet + + for carrier in ('ipv4', 'ipv6'): + with self.subTest(carrier=carrier), warnings.catch_warnings(), _Logged() as logged: + warnings.simplefilter('ignore') + data = ethernet(chains(carrier, 30, 33) if carrier == 'ipv4' else SHAPES['IPv6 in IPv6'](1000)) + info = Ethernet(data).info.to_dict() + parsing = depth(lambda: Ethernet(data)) # pylint: disable=cell-var-from-loop + rebuilding = depth(lambda: Ethernet.from_data(info)) # pylint: disable=cell-var-from-loop + self.assertLess(rebuilding - parsing, 9, (parsing, rebuilding)) + self.assertEqual(logged.messages, []) + + +if __name__ == '__main__': + unittest.main()