fix(protocols): bound a frame's dissection at 64 layers (#1610)

docs/source/contributing/pep.rst

@@ -417,6 +417,33 @@ Two traps for anyone benchmarking this library. ``reassembly=True`` and
 only the switch measures nothing; and timing several capture shapes in one
 process inflates them by up to 73%, so each shape wants its own interpreter.
 
+Recursive Dissection, With a Depth Limit
+----------------------------------------
+
+**Decided for now, open to revisit.** Each layer dissects the next from inside
+its own parse, so dissecting a frame recurses once per layer, at 9 to 12 Python
+frames a layer. Deep nesting -- IP in IP, stacked 802.1Q tags, long chains of
+IPv6 extension headers -- used to exhaust the recursion limit and leave the rest
+of the frame raw, silently, at a depth that varied with the interpreter and the
+caller's stack (:issue:`1604`, :issue:`1610`).
+
+Recursion stays, with a counted bound:
+:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` caps a frame at 64 layers,
+and :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` caps one
+chain of extension headers at 32. Past either, the rest stays
+:class:`~pcapkit.protocols.misc.raw.Raw`, with one
+:exc:`~pcapkit.utilities.warnings.ProtocolWarning`. Raising
+:func:`sys.setrecursionlimit` instead was rejected: it is process-wide, and the
+cut-off would still move with the caller's stack. The bound fits with room to
+spare, but not unlimited room. Under :program:`pytest` at the default limit of
+1,000, the costliest 64-layer frame needs a limit of 834 on Python 3.11 and 756
+on 3.14. That frame is IPv4 carrying a chain of 32 extension headers, then IPv4
+in IPv4 carrying 28 more, then TCP.
+
+An iterative model would need no such bound: a loop that dissects one layer and
+takes the next from it. It changes how every protocol hands over its payload,
+so it is a design of its own, and may replace the recursion in a future release.
+
 Logging Integration
 -------------------
 

docs/source/pcapkit/protocols/protocol.rst

@@ -73,6 +73,7 @@ Internal Definitions below.
    .. automethod:: _lookup_next_layer
    .. automethod:: _decode_next_layer
    .. automethod:: _import_next_layer
+   .. automethod:: _parse_next_layer
    .. automethod:: _get_context
 
    .. autoattribute:: _data
@@ -82,6 +83,7 @@ Internal Definitions below.
    .. autoattribute:: _protos
    .. autoattribute:: _seekset
    .. autoattribute:: _sigterm
+   .. autoattribute:: _past_layer_limit
    .. autoattribute:: __data__
 
    .. automethod:: __init__
@@ -99,6 +101,26 @@ Internal Definitions below.
    .. autoattribute:: _exproto
    .. autoattribute:: _exctx
 
+Layers per Frame
+----------------
+
+A frame is dissected to at most
+:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of its protocol
+chain, counted from the first layer the frame record carries, through tunnels,
+VLAN tags and IPv6 extension headers alike. What follows is kept as
+:class:`~pcapkit.protocols.misc.raw.Raw`, with one
+:exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1610`), and is
+marked :attr:`~pcapkit.protocols.protocol.Protocol._past_layer_limit`, so the
+default engine's TCP reassembly and flow tracing do not read a TCP segment
+there back out, as they do one the TCP parser rejected (:issue:`1518`). A
+protocol constructed from its own octets is not counted itself, so the count
+starts at the first layer it dissects. One chain of extension headers is also bounded on
+its own, by :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT`.
+The ``dpkt``, ``scapy``, ``pyshark`` and ``pypcapfile`` engines dissect with
+their own parsers, which do not apply this bound (:issue:`1609`).
+
+.. autodata:: pcapkit.protocols.protocol.FRAME_LAYER_LIMIT
+
 Data Models
 -----------
 

pcapkit/protocols/internet/ipv6.py

@@ -37,6 +37,7 @@
 from pcapkit.protocols.data.internet.ipv6 import IPv6 as Data_IPv6
 from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT
 from pcapkit.protocols.internet.ip import IP
+from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT, _frame_depth
 from pcapkit.protocols.schema.internet.ipv6 import IPv6 as Schema_IPv6
 from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length
 from pcapkit.utilities.decorators import beholder
@@ -535,6 +536,7 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None,
         _exthdr = []                # (parser class, info) per extension header
         split = None                # type: Optional[tuple[int, int]] # at first Fragment header
         opaque = False              # walk ended at a non-zero Fragment Offset
+        depth = _frame_depth.get()  # layers of the frame up to this one
 
         # traverse if next header is an extension header
         payload = self.__header__.get_payload()
@@ -553,14 +555,27 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None,
             if len(_protos) >= EXTENSION_HEADER_LIMIT:
                 break
 
+            # NOTE: Likewise past :data:`FRAME_LAYER_LIMIT
+            # <pcapkit.protocols.protocol.FRAME_LAYER_LIMIT>` layers of the
+            # frame (:issue:`1610`), since each header walked is one of them.
+            if depth + len(_protos) >= FRAME_LAYER_LIMIT:
+                break
+
             # # directly break when No Next Header occurs
             # if proto.name == 'IPv6-NoNxt':
             #     proto = None
             #     break
 
             # make protocol name
-            next_ = self._import_next_layer(proto, packet=packet, version=6, extension=True,
-                                            payload=payload)  # type: ignore[misc,call-arg,arg-type]
+            # NOTE: Counted after every header walked, as it would be were each
+            # dissected by the one before it. ESP dissects its decrypted
+            # payload even when walked, and those layers count on from here.
+            token = _frame_depth.set(depth + len(_protos))
+            try:
+                next_ = self._import_next_layer(proto, packet=packet, version=6, extension=True,
+                                                payload=payload)  # type: ignore[misc,call-arg,arg-type]
+            finally:
+                _frame_depth.reset(token)
             info = next_.info
 
             # A layer with no ``next`` field cannot safely continue the walk.
@@ -657,8 +672,12 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None,
         # ``fragment`` above, which this leaves as it was.
         ipv6_exthdr = ProtoChain.from_list(_protos)  # type: ignore[arg-type]
         self._exthdr_depth = len(_protos)  # the headers the upper layer follows
-        return super()._decode_next_layer(ipv6, None if opaque else proto, raw_len, packet=packet,
-                                          ipv6_exthdr=ipv6_exthdr, payload=payload)
+        token = _frame_depth.set(depth + len(_protos))
+        try:
+            return super()._decode_next_layer(ipv6, None if opaque else proto, raw_len, packet=packet,
+                                              ipv6_exthdr=ipv6_exthdr, payload=payload)
+        finally:
+            _frame_depth.reset(token)
 
     @beholder  # type: ignore[arg-type]
     def _import_next_layer(self, proto: 'int', length: 'Optional[int]' = None, *,  # pylint: disable=arguments-differ

pcapkit/protocols/protocol.py

@@ -16,6 +16,7 @@
 import collections
 import collections.abc
 import contextlib
+import contextvars
 import difflib
 import enum
 import functools
@@ -49,8 +50,9 @@
 from pcapkit.utilities.decorators import beholder, seekset
 from pcapkit.utilities.exceptions import (BaseError, ProtocolError, ProtocolNotFound,
                                           ProtocolNotImplemented, RegistryError, StructError,
-                                          UnsupportedCall)
-from pcapkit.utilities.warnings import RegistryWarning, UnknownFieldWarning, hold_warnings, warn
+                                          UnsupportedCall, stacklevel)
+from pcapkit.utilities.warnings import (ProtocolWarning, RegistryWarning, UnknownFieldWarning,
+                                        hold_warnings, warn)
 
 if TYPE_CHECKING:
     from enum import IntEnum as StdlibEnum
@@ -389,6 +391,37 @@ def _check_construction_keywords(cls: 'type', kwargs: 'dict[str, Any]',
          UnknownFieldWarning)
 
 
+#: Most layers dissected in one frame (:issue:`1610`). A layer is one entry of
+#: the frame's protocol chain, counted from the first layer the frame record
+#: carries, usually its link layer; a protocol constructed from its own octets
+#: is not counted itself, so the count starts at the first layer it dissects.
+#: IPv6 extension headers are layers like any other, whether
+#: :class:`~pcapkit.protocols.internet.ipv6.IPv6` walks them or each dissects
+#: the next. What follows the last layer is kept as
+#: :class:`~pcapkit.protocols.misc.raw.Raw`, with one
+#: :exc:`~pcapkit.utilities.warnings.ProtocolWarning`, unless it would have been
+#: left raw anyway or there is nothing left.
+#:
+#: Each layer dissects the next from inside its own parse, so without the bound
+#: a deep enough frame, such as IP in IP or stacked 802.1Q tags, stops wherever
+#: Python's recursion limit falls, which depends on the interpreter and on how
+#: deep the caller's own stack is. A tunnel does not restart the count, and nor
+#: does the end of a chain of extension headers, which
+#: :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` bounds on
+#: its own.
+FRAME_LAYER_LIMIT = 64
+
+#: contextvars.ContextVar[int]: How many layers of the frame being dissected
+#: come before the next one :meth:`ProtocolBase._parse_next_layer` parses, c.f.
+#: :data:`FRAME_LAYER_LIMIT`.
+_frame_depth = contextvars.ContextVar('_frame_depth', default=0)
+
+#: contextvars.ContextVar[bool]: Whether a payload is being rebuilt by
+#: :meth:`ProtocolBase._make_payload`, so that the layer it is rebuilt for
+#: parses the same octets again, c.f. :data:`FRAME_LAYER_LIMIT`.
+_rebuilding = contextvars.ContextVar('_rebuilding', default=False)
+
+
 def _short_of_hint(klass: 'Type[ProtocolBase]', captured: 'int') -> 'bool':
     """Tell whether ``captured`` octets may be too few for a ``klass`` header.
 
@@ -536,6 +569,13 @@ class ProtocolBase(Generic[_PT, _ST], metaclass=ProtocolMeta):
     #: ``object.__new__(SomeProtocol)``) sees, so reading it is always safe.
     _exctx: 'Optional[ContextRegistry]' = None
 
+    #: Whether this layer is past :data:`FRAME_LAYER_LIMIT`, and so was kept as
+    #: :class:`~pcapkit.protocols.misc.raw.Raw` by :meth:`_parse_next_layer`
+    #: without being parsed. Such a layer carries an ``error`` as one that
+    #: failed to parse does, so this is what tells the two apart: what follows
+    #: the limit is not to be read on (:issue:`1610`).
+    _past_layer_limit: 'bool' = False
+
     ##########################################################################
     # Properties.
     ##########################################################################
@@ -2017,6 +2057,14 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc
         neither. The payload is then found from the structure of ``data``
         instead, by :meth:`_lookup_payload`.
 
+        The payload is built one layer further into the frame than this
+        protocol, as :meth:`_parse_next_layer` counts it, since each layer built
+        parses its own octets again. So rebuilding a frame stops at
+        :data:`FRAME_LAYER_LIMIT` where parsing it did, rather than that many
+        layers below each layer of it (:issue:`1610`). Nor does the payload's
+        parse warn of the limit: this protocol's own parse goes over the same
+        octets, so the rebuilt frame warns of it once, as the parsed one did.
+
         Args:
             data: protocol data
             exclude: keys of ``data`` that are not the payload, though they may
@@ -2032,7 +2080,12 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc
                 from pcapkit.protocols.misc.null import \
                     NoPayload  # pylint: disable=import-outside-toplevel
                 return NoPayload()
-            return proto.from_data(data[name])
+            token, rebuilding = _frame_depth.set(_frame_depth.get() + 1), _rebuilding.set(True)
+            try:
+                return proto.from_data(data[name])
+            finally:
+                _rebuilding.reset(rebuilding)
+                _frame_depth.reset(token)
 
         proto = cast('Optional[Type[Protocol]]', data.get('__next_type__'))
         if proto is None or not (isinstance(proto, type) and issubclass(proto, ProtocolBase)):
@@ -2046,7 +2099,12 @@ def _make_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'Protoc
                 NoPayload  # pylint: disable=import-outside-toplevel
             return NoPayload()
 
-        return proto.from_data(data[name])
+        token, rebuilding = _frame_depth.set(_frame_depth.get() + 1), _rebuilding.set(True)
+        try:
+            return proto.from_data(data[name])
+        finally:
+            _rebuilding.reset(rebuilding)
+            _frame_depth.reset(token)
 
     @classmethod
     def _lookup_payload(cls, data: 'Data', exclude: 'tuple[str, ...]' = ()) -> 'tuple[Optional[Type[ProtocolBase]], Optional[str]]':  # pylint: disable=line-too-long
@@ -2288,7 +2346,8 @@ def _parse_next_layer(klass: 'Type[ProtocolBase]', file_: 'bytes', length: 'int'
 
         Returns:
             Instance of ``klass``, or of :class:`~pcapkit.protocols.misc.raw.Raw`
-            when fewer octets were captured than its header needs.
+            when fewer octets were captured than its header needs, or when it is
+            past :data:`FRAME_LAYER_LIMIT`.
 
         Note:
             :meth:`Schema.unpack <pcapkit.protocols.schema.schema.Schema.unpack>`
@@ -2317,14 +2376,44 @@ def _parse_next_layer(klass: 'Type[ProtocolBase]', file_: 'bytes', length: 'int'
             still warns. A layer captured at least as long as its hint cannot be
             replaced, and is parsed holding nothing, even inside one that holds.
 
+            A layer past :data:`FRAME_LAYER_LIMIT` in its frame is not parsed:
+            it and the rest of the frame are kept as
+            :class:`~pcapkit.protocols.misc.raw.Raw`, marked
+            :attr:`_past_layer_limit`, with one
+            :exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1610`).
+
+        Warns:
+            ProtocolWarning: If the next layer is past :data:`FRAME_LAYER_LIMIT`.
+
         """
+        depth = _frame_depth.get() + 1
+        if depth > FRAME_LAYER_LIMIT:
+            from pcapkit.protocols.misc.null import NoPayload  # isort: skip # pylint: disable=import-outside-toplevel
+            from pcapkit.protocols.misc.raw import Raw  # isort: skip # pylint: disable=import-outside-toplevel
+
+            # NOTE: Nothing is lost if the next layer is raw already, or empty.
+            if klass is not Raw and klass is not NoPayload:
+                message = (f'the frame reached the limit of {FRAME_LAYER_LIMIT} layers; the rest of it, '
+                           f'from {klass.__name__} on, is left undissected')
+                # NOTE: Inside a payload being rebuilt, the layer it is rebuilt
+                # for warns of the same octets once its own parse gets here.
+                if not _rebuilding.get():
+                    warn(message, ProtocolWarning, stacklevel=stacklevel())
+                raw = Raw(file_, length, **{**kwargs, 'error': message})
+                raw._past_layer_limit = True  # pylint: disable=protected-access
+                return raw
+
         captured = min(length, len(file_))
-        with hold_warnings(_short_of_hint(klass, captured)) as held:
-            next_ = klass(file_, length, **kwargs)  # type: ignore[abstract]
+        token = _frame_depth.set(depth)
+        try:
+            with hold_warnings(_short_of_hint(klass, captured)) as held:
+                next_ = klass(file_, length, **kwargs)  # type: ignore[abstract]
 
-            length_hint = getattr(next_, '__length_hint__', None)
-            hint = None if length_hint is None else length_hint()
-            held.discard = hint is not None and captured < hint and captured < next_.length
+                length_hint = getattr(next_, '__length_hint__', None)
+                hint = None if length_hint is None else length_hint()
+                held.discard = hint is not None and captured < hint and captured < next_.length
+        finally:
+            _frame_depth.reset(token)
         if held.discard:
             from pcapkit.protocols.misc.raw import Raw  # isort: skip # pylint: disable=import-outside-toplevel
             next_ = Raw(file_, length, **kwargs)

pcapkit/toolkit/pcap.py

@@ -118,7 +118,9 @@ def tcp_segment(frame: 'ProtocolBase') -> 'Optional[TCPSegment]':
         (:rfc:`9293#section-3.1`) if the payload of the innermost IP layer is a
         :class:`~pcapkit.protocols.misc.raw.Raw` the TCP parser rejected, of
         protocol TCP, from a datagram that is not a later fragment, with at
-        least those 20 octets captured; else :data:`None`.
+        least those 20 octets captured; else :data:`None`. A segment past
+        :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` is kept raw
+        without being parsed, not rejected, so it is not read (:issue:`1610`).
 
     Note:
         A capture snapped inside the TCP header leaves a Data Offset that runs
@@ -154,7 +156,10 @@ def tcp_segment(frame: 'ProtocolBase') -> 'Optional[TCPSegment]':
     raw = ip.payload
     # NOTE: a Raw with no error is one the caller asked for, by stopping the
     # dissection at a ``layer`` or ``protocol``, not a segment TCP rejected.
+    # Nor is one past FRAME_LAYER_LIMIT, which carries an error all the same:
+    # the frame is not dissected past it, so its TCP is not read (#1610).
     if (not isinstance(raw, Raw) or raw.info.error is None
+            or raw._past_layer_limit  # pylint: disable=protected-access
             or ip_info.protocol != Enum_TransType.TCP):
         return None