fix(toolkit): dpkt and scapy find TCP in IPv4 where the default engine does (#1606, #1617)

pcapkit/toolkit/dpkt.py

@@ -23,7 +23,8 @@
 from pcapkit.foundation.traceflow.data.data import FrameRecord
 from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet
 from pcapkit.protocols.data.misc.pcap.frame import FrameInfo as Data_FrameInfo
-from pcapkit.toolkit.pypcapfile import _default_accepts
+from pcapkit.toolkit.pypcapfile import (IPV4_EXTENSION_HEADERS, TCP_MIN_HEADER_LEN,
+                                        _default_accepts, _past_extension_headers)
 from pcapkit.utilities.exceptions import ProtocolError, UnsupportedCall
 
 if TYPE_CHECKING:
@@ -462,6 +463,112 @@ def _ipv4_accepted(ipv4: 'IP') -> 'bool':
     return _default_accepts('IPv4', ipv4.pack_hdr() + bytes(ipv4.opts))
 
 
+def _past_ipv4_extension_headers(packet: 'Packet', ipv4: 'IP') -> 'Optional[tuple[int, bytes]]':
+    """Step over the headers at the head of a DPKT IPv4 packet's payload that the default engine reads past.
+
+    Args:
+        packet: DPKT packet ``ipv4`` is a layer of, whose octets
+            :func:`_layer2bytes` slices.
+        ipv4: DPKT IPv4 packet, whose protocol is one of the
+            :data:`~pcapkit.toolkit.pypcapfile.IPV4_EXTENSION_HEADERS`.
+
+    Returns:
+        The protocol number after the last of those headers and the octets from
+        its header on, as far as the Total Length goes -- a Total Length of
+        ``0``, which TCP segmentation offload leaves, runs to the end of the
+        frame -- or :data:`None` if the default engine reads no upper-layer
+        header there: ``ipv4`` is a later fragment, whose data is a slice of the
+        datagram's payload (:rfc:`791#section-3.2`), or it leaves one of the
+        headers undissected, which
+        :func:`~pcapkit.toolkit.pypcapfile._past_extension_headers` decides by
+        the default engine's own rules, up to
+        :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT`
+        headers in a row (:issue:`1604`).
+
+    Note:
+        The default engine dispatches an IPv4 packet's protocol through the
+        registry IPv6 uses, so it reads an AH or IPv6 extension header there
+        and what its Next Header names. `DPKT`_ decodes protocol 0 as IPv4 and
+        an AH as :class:`dpkt.ah.AH`, and keeps the rest as octets, so the
+        octets are read instead (:issue:`1606`).
+
+    .. _DPKT: https://dpkt.readthedocs.io
+
+    """
+    if ipv4.offset:
+        return None
+    wire, _ = _layer2bytes(packet, ipv4)
+    ihl = ipv4.hl * 4
+    return _past_extension_headers(ipv4.p, wire[ihl:max(ipv4.len, ihl)] if ipv4.len else wire[ihl:])
+
+
+def _decode_tcp(segment: 'bytes') -> 'Optional[TCP]':
+    """Decode a TCP segment the default engine reads behind an AH or IPv6 extension header in IPv4.
+
+    Args:
+        segment: Raw octets from the TCP header on, as far as the IPv4 packet
+            carrying it goes (see :func:`_past_ipv4_extension_headers`).
+
+    Returns:
+        The segment, with ``segment`` attached by :func:`attach_buffer` so that
+        its header is the captured one; or :data:`None` if the default engine
+        reads no TCP there: fewer than :data:`TCP_MIN_HEADER_LEN
+        <pcapkit.toolkit.pypcapfile.TCP_MIN_HEADER_LEN>` octets, a Data Offset
+        counting fewer or more than were captured, or options its TCP parser
+        rejects (see :func:`~pcapkit.toolkit.pypcapfile._default_accepts`).
+
+    Note:
+        Directly over IP, the default engine reads a TCP header its parser
+        rejects out of the raw octets left behind
+        (:func:`pcapkit.toolkit.pcap.tcp_segment`), but behind another header
+        it does not, so the parser's answer stands here, as for the pypcapfile
+        engine.
+
+    """
+    import dpkt  # isort:skip
+
+    hdr_len = (segment[12] >> 4) * 4 if len(segment) >= TCP_MIN_HEADER_LEN else 0
+    if hdr_len < TCP_MIN_HEADER_LEN or hdr_len > len(segment):
+        return None
+    if hdr_len > TCP_MIN_HEADER_LEN and not _default_accepts('TCP', segment[:hdr_len]):
+        return None
+    tcp = dpkt.tcp.TCP(segment)
+    attach_buffer(tcp, segment)
+    return tcp
+
+
+def _decode_ip(protocol: 'int', data: 'bytes') -> 'Optional[IP | IP6]':
+    """Decode an IP packet the default engine reads behind an AH or IPv6 extension header in IPv4.
+
+    Args:
+        protocol: The IP-in-IP protocol number, one of the :data:`IP_TUNNELS`.
+        data: Raw octets from the IP header on, as far as the IPv4 packet
+            carrying it goes (see :func:`_past_ipv4_extension_headers`).
+
+    Returns:
+        The packet as `DPKT`_ decodes one in a tunnel, with ``data`` attached by
+        :func:`attach_buffer`, so that :func:`_layer2bytes` slices its layers
+        out of the captured octets; or :data:`None` if decoding it raises
+        anything at all, as :class:`~pcapkit.foundation.engines.dpkt.DPKT` then
+        keeps a frame raw. `DPKT`_ raises more than its own
+        :exc:`~dpkt.dpkt.UnpackError`: dpkt 1.9.8 reads ``frag_off`` from
+        whatever header follows an IPv6 Fragment header, so one followed by
+        another extension header raises :exc:`AttributeError` (:issue:`1351`).
+
+    .. _DPKT: https://dpkt.readthedocs.io
+
+    """
+    import dpkt  # isort:skip
+
+    klass = dpkt.ip.IP if protocol == Enum_TransType.IPv4 else dpkt.ip6.IP6
+    try:
+        ip = klass(data)
+    except Exception:  # as the DPKT engine does, for a whole frame
+        return None
+    attach_buffer(ip, data)
+    return ip
+
+
 def _tcp_carrier(packet: 'Packet') -> 'tuple[Optional[IP | IP6], Optional[TCP]]':
     """Fetch the IP layer of a DPKT packet that carries its TCP segment, and the segment.
 
@@ -476,14 +583,20 @@ def _tcp_carrier(packet: 'Packet') -> 'tuple[Optional[IP | IP6], Optional[TCP]]'
         rejects the header of an IPv4 packet on the way (see
         :func:`_ipv4_accepted`). A tunnel is a packet whose protocol, or for
         IPv6 the next header after its extension headers, is one of the
-        :data:`IP_TUNNELS`.
+        :data:`IP_TUNNELS`. An IPv4 packet whose protocol names an AH or IPv6
+        extension header carries what the default engine reads past them (see
+        :func:`_past_ipv4_extension_headers`), a tunnel or a TCP segment, which
+        are decoded from the captured octets (see :func:`_decode_ip` and
+        :func:`_decode_tcp`).
 
     Note:
         `DPKT`_ decodes an IPv4 or IPv6 packet inside another (:rfc:`2003`,
         :rfc:`2473`, :rfc:`4213`) as its ``data``. The TCP endpoints are the
         innermost packet's addresses, as for the default engine
         (:func:`pcapkit.toolkit.pcap.tcp_segment`); the outer ones are the
-        tunnel's (:issue:`1581`).
+        tunnel's (:issue:`1581`). A segment found behind an extension header is
+        no layer of ``packet``, so it carries its own octets, which
+        :func:`_layer2bytes` then hands back (:issue:`1606`).
 
     .. _DPKT: https://dpkt.readthedocs.io
 
@@ -493,16 +606,28 @@ def _tcp_carrier(packet: 'Packet') -> 'tuple[Optional[IP | IP6], Optional[TCP]]'
     ip = _network(packet)
     if ip is None:
         return None, None
+    root = packet   # the packet ``ip`` is a layer of
     while True:
-        if isinstance(ip, dpkt.ip.IP) and not _ipv4_accepted(ip):
-            return None, None
+        if isinstance(ip, dpkt.ip.IP):
+            if not _ipv4_accepted(ip):
+                return None, None
+            if ip.p in IPV4_EXTENSION_HEADERS:
+                upper = _past_ipv4_extension_headers(root, ip)
+                if upper is not None and upper[0] == Enum_TransType.TCP:
+                    return ip, _decode_tcp(upper[1])
+                if upper is None or upper[0] not in IP_TUNNELS or (inner := _decode_ip(*upper)) is None:
+                    return ip, None
+                root = ip = inner
+                continue
         if getattr(ip, 'p', None) not in IP_TUNNELS or not isinstance(ip.data, (dpkt.ip.IP, dpkt.ip6.IP6)):
             break
         ip = ip.data
 
     tcp = getattr(ip, 'tcp', None)  # type: Optional[TCP]
     if tcp is None and type(getattr(ip, 'data', None)).__name__ == 'TCP':
         tcp = cast('TCP', ip.data)
+    if tcp is not None and root is not packet:
+        attach_buffer(tcp, _layer2bytes(root, tcp)[0])
     return ip, tcp
 
 
@@ -542,12 +667,13 @@ def packet2chain(packet: 'Packet') -> 'str':
         packet: DPKT packet.
 
     Returns:
-        Colon (``:``) separated list of protocol chain.
+        Colon (``:``) separated list of protocol chain. It ends at the first
+        payload that is no layer, as in :func:`packet2dict`, e.g. VRRP's ``''``.
 
     """
     chain = [type(packet).__name__]
     payload = packet.data
-    while not isinstance(payload, bytes):
+    while hasattr(payload, '__hdr_fields__'):
         chain.append(type(payload).__name__)
         payload = payload.data
     return ':'.join(chain)
@@ -563,15 +689,21 @@ def packet2dict(packet: 'Packet', timestamp: 'float', *,
         data_link: Data link type.
 
     Returns:
-        Dict[str, Any]: A :obj:`dict` mapping of packet data.
+        Dict[str, Any]: A :obj:`dict` mapping of packet data. A layer's payload
+        is mapped as a layer of its own only where it is one, i.e. has the
+        header fields a :class:`dpkt.dpkt.Packet` has; any other is leaf data.
+        `DPKT`_ leaves some undecoded payloads as a :obj:`str` rather than as
+        :obj:`bytes`, e.g. ``''`` for VRRP's.
+
+    .. _DPKT: https://dpkt.readthedocs.io
 
     """
     def wrapper(packet: 'Packet') -> 'dict[str, Any]':
         dict_ = {}  # type: dict[str, Any]
         for field in packet.__hdr_fields__:
             dict_[field] = getattr(packet, field, None)
         payload = packet.data
-        if not isinstance(payload, bytes):
+        if hasattr(payload, '__hdr_fields__'):
             dict_[type(payload).__name__] = wrapper(payload)
         return dict_
 

pcapkit/toolkit/scapy.py

@@ -47,7 +47,9 @@
 from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet
 from pcapkit.protocols.data.misc.pcap.frame import FrameInfo as Data_FrameInfo
 from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length
-from pcapkit.toolkit.pypcapfile import _default_accepts
+from pcapkit.toolkit.pypcapfile import (IP_TUNNELS, IPV4_EXTENSION_HEADERS, IPV4_HEADER_LEN,
+                                        IPV6_HEADER_LEN, TCP_MIN_HEADER_LEN, _default_accepts,
+                                        _past_extension_headers)
 from pcapkit.utilities.compat import ModuleNotFoundError  # pylint: disable=redefined-builtin
 from pcapkit.utilities.exceptions import MissingKeyError, ModuleNotFound, stacklevel
 from pcapkit.utilities.warnings import ScapyWarning, warn
@@ -332,17 +334,62 @@ def _ipv4_accepted(ipv4: 'IP') -> 'bool':
     return _default_accepts('IPv4', octets[:max((octets[0] & 0x0F) * 4, 20)])
 
 
-def _carrier(layer: 'Packet') -> 'IP | IPv6 | None':
-    """Fetch the IP layer that carries ``layer``.
+def _tcp_behind_extension_headers(segment: 'bytes') -> 'tuple[bytes, bytes] | None':
+    """Split a TCP segment the default engine reads behind an AH or IPv6 extension header in IPv4.
 
     Args:
-        layer: A layer of a dissected Scapy packet.
+        segment: Raw octets from the TCP header on, as far as the IPv4 layer
+            carrying it goes (see :func:`_tcp_carrier`).
+
+    Returns:
+        The header, as many octets as the Data Offset counts, and the payload
+        after it; or :data:`None` if the default engine reads no TCP there:
+        fewer than :data:`TCP_MIN_HEADER_LEN
+        <pcapkit.toolkit.pypcapfile.TCP_MIN_HEADER_LEN>` octets, a Data Offset
+        counting fewer or more than were captured, or options its TCP parser
+        rejects (see :func:`~pcapkit.toolkit.pypcapfile._default_accepts`).
+
+    Note:
+        Directly over IP, the default engine reads a TCP header its parser
+        rejects out of the raw octets left behind
+        (:func:`pcapkit.toolkit.pcap.tcp_segment`), but behind another header
+        it does not, so the parser's answer stands here, as for the pypcapfile
+        engine.
+
+    """
+    hdr_len = (segment[12] >> 4) * 4 if len(segment) >= TCP_MIN_HEADER_LEN else 0
+    if hdr_len < TCP_MIN_HEADER_LEN or hdr_len > len(segment):
+        return None
+    if hdr_len > TCP_MIN_HEADER_LEN and not _default_accepts('TCP', segment[:hdr_len]):
+        return None
+    return segment[:hdr_len], segment[hdr_len:]
+
+
+def _tcp_carrier(packet: 'Packet') -> 'tuple[IP | IPv6, TCP, bytes, bytes] | None':
+    """Fetch the TCP segment of a Scapy packet, and the IP layer that carries it.
+
+    Args:
+        packet: Scapy packet.
 
     Returns:
         The nearest :class:`~scapy.layers.inet.IP` or
-        :class:`~scapy.layers.inet6.IPv6` layer above ``layer``, or
-        :data:`None` if there is none, or the default engine rejects the
-        header of an IPv4 layer above it (see :func:`_ipv4_accepted`).
+        :class:`~scapy.layers.inet6.IPv6` layer above the packet's TCP layer,
+        that layer, and the segment's header and payload (see
+        :func:`_tcp_segment`); or :data:`None` if there is no such IP layer, or
+        the default engine reads no TCP there: an IPv4 layer above it has a
+        header that engine rejects (see :func:`_ipv4_accepted`), or is a later
+        fragment, whose data is a slice of the datagram's payload with no
+        upper-layer header in it (:rfc:`791#section-3.2`, :issue:`1617`).
+
+        An IPv4 layer whose protocol names an AH or IPv6 extension header
+        carries what the default engine reads past those headers, by its own
+        rules and up to
+        :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` of
+        them in a row (see
+        :func:`~pcapkit.toolkit.pypcapfile._past_extension_headers`): a tunnel,
+        walked on as above, or a TCP segment (see
+        :func:`_tcp_behind_extension_headers`), each dissected from the
+        octets (:issue:`1606`).
 
     Note:
         The nearest is the innermost IP layer of an IP-in-IP tunnel
@@ -351,22 +398,51 @@ def _carrier(layer: 'Packet') -> 'IP | IPv6 | None':
         (:func:`pcapkit.toolkit.pcap.tcp_segment`). ``packet['IP']`` finds the
         outermost IPv4 layer instead, which is the tunnel's, or with IPv4
         carried in IPv6 the inner one (:issue:`1581`). The extension header
-        layers `Scapy`_ dissects between an IP layer and its payload are not
+        layers `Scapy`_ dissects between an IPv6 layer and its payload are not
         IP layers, so are stepped over.
 
+        Inside IPv4, `Scapy`_ dissects an AH alone of those headers, and behind
+        one only IPv6 and, for a Next Header of ``0``, IPv4, so the default
+        engine's rules are applied to the octets instead. It also dissects IPv6
+        and AH inside a later fragment, binding neither to a zero fragment
+        offset as it binds TCP.
+
     """
-    from scapy.layers.inet import IP
+    from scapy.layers.inet import IP, TCP, IPerror
     from scapy.layers.inet6 import IPv6
+    from scapy.packet import NoPayload
 
+    tcp = cast('TCP | None', packet.getlayer('TCP'))
     carrier = None  # type: IP | IPv6 | None
-    above = layer.underlayer
-    while above is not None:
-        if isinstance(above, IP) and not _ipv4_accepted(above):
+    layer = packet  # type: Packet
+    while layer is not tcp:
+        if isinstance(layer, NoPayload):
             return None
-        if carrier is None and isinstance(above, (IP, IPv6)):
-            carrier = above
-        above = above.underlayer
-    return carrier
+        if isinstance(layer, IP):
+            if not _ipv4_accepted(layer) or layer.frag:
+                return None
+            carrier = layer
+            # not the datagram an ICMP error quotes, which the default engine does not dissect
+            if layer.proto in IPV4_EXTENSION_HEADERS and not isinstance(layer, IPerror):
+                upper = _past_extension_headers(layer.proto, _within_datagrams(layer.payload))
+                if upper is not None and upper[0] == Enum_TransType.TCP:
+                    segment = _tcp_behind_extension_headers(upper[1])
+                    return None if segment is None else (carrier, TCP(upper[1]), *segment)
+                if upper is None or upper[0] not in IP_TUNNELS:
+                    return None
+                version, data = IP_TUNNELS[upper[0]], upper[1]
+                if len(data) < (IPV4_HEADER_LEN if version == 4 else IPV6_HEADER_LEN):
+                    return None
+                layer = IP(data) if version == 4 else IPv6(data)
+                tcp = cast('TCP | None', layer.getlayer('TCP'))
+                continue
+        elif isinstance(layer, IPv6):
+            carrier = layer
+        layer = layer.payload
+
+    if carrier is None or (segment := _tcp_segment(tcp)) is None:
+        return None
+    return carrier, tcp, *segment
 
 
 def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Address] | None':
@@ -512,20 +588,19 @@ def tcp_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'TCP_Packet | None
         * If the ``packet`` can be used for TCP reassembly. A packet can be reassembled
           if it contains TCP layer (:class:`scapy.layers.inet.TCP`) whose fixed header
           lies inside its datagram, and whose Data Offset counts at least that header,
-          behind no IPv4 header the default engine rejects. It is keyed by the IP layer
-          that carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_carrier`).
+          behind no IPv4 header the default engine rejects and in no later IPv4 fragment,
+          or a TCP segment the default engine reads behind an AH or IPv6 extension header
+          in IPv4. It is keyed by the IP layer that carries it -- in an IP-in-IP tunnel,
+          the innermost (see :func:`_tcp_carrier`).
         * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP
           reassembly (:term:`reasm.tcp.packet`) will be returned; otherwise, returns :data:`None`.
 
     See Also:
         :class:`pcapkit.foundation.reassembly.tcp.TCP`
 
     """
-    if 'TCP' in packet:
-        tcp = cast('TCP', packet['TCP'])
-        if (ip := _carrier(tcp)) is None or (segment := _tcp_segment(tcp)) is None:
-            return None
-        header, payload = segment
+    if (found := _tcp_carrier(packet)) is not None:
+        ip, tcp, header, payload = found
 
         raw_len = len(payload)                      # payload length, header excludes
         data = TCP_Packet(
@@ -572,11 +647,8 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | No
         :class:`pcapkit.foundation.traceflow.tcp.TCP`
 
     """
-    if 'TCP' in packet:
-        tcp = cast('TCP', packet['TCP'])
-        if (ip := _carrier(tcp)) is None or (segment := _tcp_segment(tcp)) is None:
-            return None
-        header, payload = segment
+    if (found := _tcp_carrier(packet)) is not None:
+        ip, tcp, header, payload = found
 
         # NOTE: the link type of the interface the packet was read from, which
         # :class:`~pcapkit.foundation.engines.scapy.Scapy` attaches with