fix(toolkit): dpkt and scapy check IPv4 headers as the default engine does (#1607)

pcapkit/toolkit/dpkt.py

@@ -23,6 +23,7 @@
 from pcapkit.foundation.traceflow.data.data import FrameRecord
 from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet
 from pcapkit.protocols.data.misc.pcap.frame import FrameInfo as Data_FrameInfo
+from pcapkit.toolkit.pypcapfile import _default_accepts
 from pcapkit.utilities.exceptions import ProtocolError, UnsupportedCall
 
 if TYPE_CHECKING:
@@ -431,6 +432,36 @@ def _network(packet: 'Packet') -> 'Optional[IP | IP6]':
     return getattr(packet, 'ip6', None)
 
 
+def _ipv4_accepted(ipv4: 'IP') -> 'bool':
+    """Test if the default engine reads a DPKT IPv4 packet as an IPv4 layer.
+
+    Args:
+        ipv4: DPKT IPv4 packet.
+
+    Returns:
+        :data:`False` if the default engine's IPv4 parser rejects its header --
+        its version is not 4, its options are ones the parser rejects, or its
+        IHL runs past the captured octets (:issue:`1607`) -- as the default
+        engine then dissects nothing past it; else :data:`True`.
+
+    Note:
+        `DPKT`_ decodes what an IPv4 EtherType or tunnel carries as IPv4
+        whatever its version, and never checks its options, so the header is
+        asked of the parser itself, as the pypcapfile engine asks it (see
+        :func:`pcapkit.toolkit.pypcapfile._default_accepts`). Its octets are the
+        fields as `DPKT`_ read them and the options as captured, which is
+        exact. A header of version 4 with no options is not parsed: `DPKT`_
+        decodes none shorter than 20 octets or with an IHL below 5, and the
+        parser checks nothing else of one.
+
+    .. _DPKT: https://dpkt.readthedocs.io
+
+    """
+    if ipv4.v == 4 and ipv4.hl == 5:
+        return True
+    return _default_accepts('IPv4', ipv4.pack_hdr() + bytes(ipv4.opts))
+
+
 def _tcp_carrier(packet: 'Packet') -> 'tuple[Optional[IP | IP6], Optional[TCP]]':
     """Fetch the IP layer of a DPKT packet that carries its TCP segment, and the segment.
 
@@ -441,9 +472,11 @@ def _tcp_carrier(packet: 'Packet') -> 'tuple[Optional[IP | IP6], Optional[TCP]]'
         The innermost IPv4 or IPv6 packet of the network layer (see
         :func:`_network`) and of the IP-in-IP tunnels it carries, and its
         :class:`dpkt.tcp.TCP` payload, if it has one; else :data:`None` for the
-        latter, and for both with no network layer. A tunnel is a packet whose
-        protocol, or for IPv6 the next header after its extension headers, is
-        one of the :data:`IP_TUNNELS`.
+        latter, and for both with no network layer, or if the default engine
+        rejects the header of an IPv4 packet on the way (see
+        :func:`_ipv4_accepted`). A tunnel is a packet whose protocol, or for
+        IPv6 the next header after its extension headers, is one of the
+        :data:`IP_TUNNELS`.
 
     Note:
         `DPKT`_ decodes an IPv4 or IPv6 packet inside another (:rfc:`2003`,
@@ -460,7 +493,11 @@ def _tcp_carrier(packet: 'Packet') -> 'tuple[Optional[IP | IP6], Optional[TCP]]'
     ip = _network(packet)
     if ip is None:
         return None, None
-    while getattr(ip, 'p', None) in IP_TUNNELS and isinstance(ip.data, (dpkt.ip.IP, dpkt.ip6.IP6)):
+    while True:
+        if isinstance(ip, dpkt.ip.IP) and not _ipv4_accepted(ip):
+            return None, None
+        if getattr(ip, 'p', None) not in IP_TUNNELS or not isinstance(ip.data, (dpkt.ip.IP, dpkt.ip6.IP6)):
+            break
         ip = ip.data
 
     tcp = getattr(ip, 'tcp', None)  # type: Optional[TCP]
@@ -565,7 +602,8 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *,
         Data for IPv4 reassembly.
 
         * If the ``packet`` can be used for IPv4 reassembly. A packet can be reassembled
-          if it contains IPv4 layer (:class:`dpkt.ip.IP`) and the **DF** (:attr:`dpkt.ip.IP.df`)
+          if it contains IPv4 layer (:class:`dpkt.ip.IP`) whose header the default engine
+          accepts (see :func:`_ipv4_accepted`), and the **DF** (:attr:`dpkt.ip.IP.df`)
           flag is :data:`False`.
         * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for IPv4
           reassembly (:term:`reasm.ipv4.packet`) will be returned; otherwise, returns :data:`None`.
@@ -586,7 +624,7 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *,
     network = _network(packet)
     # an IPv6 layer is the one with extension headers, as in ``_wire_hdr_len``
     ipv4 = None if hasattr(network, 'extension_hdrs') else network
-    if ipv4 is not None:
+    if ipv4 is not None and _ipv4_accepted(ipv4):
         if ipv4.df:     # dismiss not fragmented packet
             return None
         # internet header length, in octets -- ``IP.hl`` counts 32-bit words and
@@ -720,7 +758,8 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *,
 
         * If the ``packet`` can be used for TCP reassembly. A packet can be reassembled
           if it contains TCP layer (:class:`dpkt.tcp.TCP`), keyed by the IP packet that
-          carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_tcp_carrier`).
+          carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_tcp_carrier`)
+          -- and behind no IPv4 header the default engine rejects.
         * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP
           reassembly (:term:`reasm.tcp.packet`) will be returned; otherwise, returns :data:`None`.
 
@@ -785,7 +824,8 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *,
 
         * If the ``packet`` can be used for TCP flow tracing. A packet can be reassembled
           if it contains TCP layer (:class:`dpkt.tcp.TCP`), keyed by the IP packet that
-          carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_tcp_carrier`).
+          carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_tcp_carrier`)
+          -- and behind no IPv4 header the default engine rejects.
         * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP
           flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`.
 

pcapkit/toolkit/scapy.py

@@ -47,6 +47,7 @@
 from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet
 from pcapkit.protocols.data.misc.pcap.frame import FrameInfo as Data_FrameInfo
 from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length
+from pcapkit.toolkit.pypcapfile import _default_accepts
 from pcapkit.utilities.compat import ModuleNotFoundError  # pylint: disable=redefined-builtin
 from pcapkit.utilities.exceptions import MissingKeyError, ModuleNotFound, stacklevel
 from pcapkit.utilities.warnings import ScapyWarning, warn
@@ -301,6 +302,36 @@ def _tcp_segment(tcp: 'TCP') -> 'tuple[bytes, bytes] | None':
     return segment[:hdr_len], segment[hdr_len:]
 
 
+def _ipv4_accepted(ipv4: 'IP') -> 'bool':
+    """Test if the default engine reads a Scapy IPv4 layer as an IPv4 layer.
+
+    Args:
+        ipv4: IPv4 layer of a dissected Scapy packet.
+
+    Returns:
+        :data:`False` if the default engine's IPv4 parser rejects its header --
+        its version is not 4, its IHL is below 5 or runs past the captured
+        octets, or its options are ones the parser rejects (:issue:`1607`) --
+        as the default engine then dissects nothing past it; else :data:`True`.
+
+    Note:
+        `Scapy`_ dissects what an IPv4 EtherType or tunnel carries as IPv4
+        whatever its version or IHL, and never checks its options, so the
+        header is asked of the parser itself, as the pypcapfile engine asks it
+        (see :func:`pcapkit.toolkit.pypcapfile._default_accepts`). Its octets
+        are those `Scapy`_ dissected the layer from, which ``bytes()`` gives
+        back as they were, as far as the IHL goes, and no fewer than the 20 of
+        the fixed header, as the default engine reads at least those. A header
+        of version 4 with an IHL of 5 is not parsed: `Scapy`_ dissects none
+        shorter than 20 octets, and the parser checks nothing else of one.
+
+    """
+    if ipv4.version == 4 and ipv4.ihl == 5:
+        return True
+    octets = bytes(ipv4)
+    return _default_accepts('IPv4', octets[:max((octets[0] & 0x0F) * 4, 20)])
+
+
 def _carrier(layer: 'Packet') -> 'IP | IPv6 | None':
     """Fetch the IP layer that carries ``layer``.
 
@@ -310,7 +341,8 @@ def _carrier(layer: 'Packet') -> 'IP | IPv6 | None':
     Returns:
         The nearest :class:`~scapy.layers.inet.IP` or
         :class:`~scapy.layers.inet6.IPv6` layer above ``layer``, or
-        :data:`None` if there is none.
+        :data:`None` if there is none, or the default engine rejects the
+        header of an IPv4 layer above it (see :func:`_ipv4_accepted`).
 
     Note:
         The nearest is the innermost IP layer of an IP-in-IP tunnel
@@ -326,9 +358,14 @@ def _carrier(layer: 'Packet') -> 'IP | IPv6 | None':
     from scapy.layers.inet import IP
     from scapy.layers.inet6 import IPv6
 
-    carrier = layer.underlayer
-    while carrier is not None and not isinstance(carrier, (IP, IPv6)):
-        carrier = carrier.underlayer
+    carrier = None  # type: IP | IPv6 | None
+    above = layer.underlayer
+    while above is not None:
+        if isinstance(above, IP) and not _ipv4_accepted(above):
+            return None
+        if carrier is None and isinstance(above, (IP, IPv6)):
+            carrier = above
+        above = above.underlayer
     return carrier
 
 
@@ -343,7 +380,8 @@ def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Ad
         Data for IPv4 reassembly.
 
         * If the ``packet`` can be used for IPv4 reassembly. A packet can be reassembled
-          if it contains IPv4 layer (:class:`scapy.layers.inet.IP`) and the **DF**
+          if it contains IPv4 layer (:class:`scapy.layers.inet.IP`) whose header the
+          default engine accepts (see :func:`_ipv4_accepted`), and the **DF**
           (:attr:`scapy.layers.inet.IP.flags.DF`) flag is :data:`False`.
         * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for IPv4
           reassembly (:term:`reasm.ipv4.packet`) will be returned; otherwise, returns :data:`None`.
@@ -356,6 +394,8 @@ def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Ad
         ipv4 = cast('IP', packet['IP'])
         if ipv4.flags.DF:       # dismiss not fragmented packet
             return None
+        if not _ipv4_accepted(ipv4):
+            return None
 
         data = IP_Packet(
             bufid=(
@@ -471,9 +511,9 @@ def tcp_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'TCP_Packet | None
 
         * If the ``packet`` can be used for TCP reassembly. A packet can be reassembled
           if it contains TCP layer (:class:`scapy.layers.inet.TCP`) whose fixed header
-          lies inside its datagram, and whose Data Offset counts at least that header.
-          It is keyed by the IP layer that carries it -- in an IP-in-IP tunnel, the
-          innermost (see :func:`_carrier`).
+          lies inside its datagram, and whose Data Offset counts at least that header,
+          behind no IPv4 header the default engine rejects. It is keyed by the IP layer
+          that carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_carrier`).
         * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP
           reassembly (:term:`reasm.tcp.packet`) will be returned; otherwise, returns :data:`None`.