fix(toolkit): dpkt walks IP layers off the octets as the default engine does (#1609, #1626, #1651, #1654)

pcapkit/foundation/engines/dpkt.py

@@ -476,9 +476,9 @@ def read_frame(self) -> 'DPKTPacket':
             for more operational information.
 
         """
-        from pcapkit.toolkit.dpkt import (attach_buffer, attach_timestamp, ipv4_reassembly,
-                                          ipv6_reassembly, packet2dict, tcp_reassembly,
-                                          tcp_traceflow)
+        from pcapkit.toolkit.dpkt import (attach_buffer, attach_linktype, attach_timestamp,
+                                          ipv4_reassembly, ipv6_reassembly, packet2dict,
+                                          tcp_reassembly, tcp_traceflow)
         ext = self._extractor
 
         reader = self._extmp
@@ -514,6 +514,9 @@ def read_frame(self) -> 'DPKTPacket':
         # it again recomputes zeroed checksums and lengths into it, so keep those
         # too for the adapters to slice the wire octets out of.
         attach_buffer(packet, pkt)
+        # And a frame kept as raw data does not say what link layer its octets
+        # start with, which the adapters read them by (#1626).
+        attach_linktype(packet, linktype)
 
         # verbose output
         ext._frnum += 1

tests/foundation/engines/test_ext_chain_limit_agreement_1604_unit.py

@@ -13,11 +13,12 @@
 ``pypcapfile`` walks the chain itself and stops at the same depth. It leaves TCP
 over IPv6 out, so it is compared over IPv4 only, and ``declined`` checks that
 it warns of leaving out TCP over IPv6 exactly where ``default`` finds some. It
-imports on Python 3.11 only, and is skipped elsewhere.
+imports on Python 3.11 only, and is skipped elsewhere. ``dpkt``'s adapters walk
+the chain off the octets, and stop at the same depth too (#1609).
 
 The other parsers walk the chain themselves, and cannot be told where to stop,
 so where they find the TCP past the limit is recorded in :attr:`KNOWN_FAILURES`
-under #1609 rather than hidden. Measured on dpkt 1.9.8, scapy 2.7.0 and tshark 4.6.9.
+under #1609 rather than hidden. Measured on scapy 2.7.0 and tshark 4.6.9.
 
 Every capture is built in a temporary directory, so no fixture is read.
 
@@ -155,9 +156,6 @@ class TestExtensionHeaderChainAgreement(harness.RoundTripBase):
 
     STATUSES = ('ERROR', 'INPUTS', 'DECLINED')
     KNOWN_FAILURES = (
-        Gap(1609, 'dpkt walks the whole extension header chain in a loop (dpkt.ip6.IP6.unpack), '
-                  'so finds the TCP past EXTENSION_HEADER_LIMIT that the default engine leaves raw',
-            'INPUTS', 'extra, default has none', ('ipv6-33/dpkt/*', 'ipv6-1000/dpkt/*')),
         Gap(1609, 'scapy dissects each extension header behind the last, with no limit of its own, '
                   'so finds the TCP past EXTENSION_HEADER_LIMIT; it gives up near 245 layers at the '
                   'default recursion limit, and so agrees at 1,000',

tests/toolkit/test_dpkt_reassembly_rawbytes_unit.py

@@ -12,10 +12,9 @@
 ``len()``: :meth:`dpkt.ip6.IP6.__len__` sums ``len()`` of the extension headers
 rather than their wire lengths, which is 12 octets plus the rest of the frame for
 an Authentication header and 8 octets short for a Routing header whose Hdr Ext
-Len is odd. :class:`~dpkt.ip6.IP6ExtensionHeader` records the wire length in
-``length``, which is what :meth:`dpkt.ip6.IP6.unpack` walks the buffer by, so
-that is what the offset is built from -- and the slice is then checked against
-the layer's own header before it is handed back.
+Len is odd. So the extension headers are walked off the octets, by the lengths
+their own fields declare (#1626), and the network layer's slice is checked
+against its own header before anything is read from it.
 
 Every frame below is written to a PCAP file and read back through the engine, so
 the tests exercise the path a capture actually takes rather than a packet with a
@@ -357,10 +356,11 @@ def test_the_unbuffered_fallback_returns_dpkts_serialisation(self) -> None:
         self.assertEqual(bytes(data.header) + bytes(data.payload), IPV4_TCP_REPACKED)
         self.assertNotEqual(IPV4_TCP_REPACKED, IPV4_TCP)
 
-        # the TCP layer alone round-trips: dpkt.tcp.TCP.__bytes__ recomputes
-        # nothing, the pseudo-header checksum being dpkt.ip.IP.__bytes__'s doing
+        # the TCP segment is read out of the same serialisation of the network
+        # layer, so it carries the checksum dpkt.ip.IP.__bytes__ fills in with
+        # the IPv4 header's (#1626)
         tcp = toolkit.tcp_reassembly(dpkt.ethernet.Ethernet(ETH_IPV4 + IPV4_TCP), 0.0)
-        self.assertEqual(bytes(tcp.header), IPV4_TCP[20:40])
+        self.assertEqual(bytes(tcp.header), IPV4_TCP_REPACKED[20:40])
 
     def test_attaching_the_buffer_makes_the_octets_the_wires_again(self) -> None:
         import dpkt

tests/toolkit/test_dpkt_unit.py

@@ -76,42 +76,27 @@ def pack(self) -> bytes:
         return b'I' * 40 + b'H' * 8 + b'F' * 8 + b'PAYLOAD'
 
 
-class TCP:
-    __hdr_fields__ = ('sport', 'dport', 'seq', 'ack', 'flags')
-    __hdr_len__ = 4
-    #: Data Offset, in 32-bit words. ``off * 4`` has to agree with the header
-    #: portion of :meth:`pack`, the way it does on a real :class:`dpkt.tcp.TCP`.
-    off = 1
-    sport = 2345
-    dport = 443
-    seq = 20
-    ack = 15
-    flags = 0b00010011
-    data = b'ip6'
-
-    def pack(self) -> bytes:
-        return b'HEADip6'
-
-
-class FakeTCPIPv6:
-    __hdr_fields__ = ('src', 'dst')
-    src = ip_address('2001:db8::10').packed
-    dst = ip_address('2001:db8::20').packed
-
-    def __init__(self) -> None:
-        self.data = TCP()
+def _ipv6_frame(payload: bytes, nxt: int, *, src: str, dst: str, flow: int = 0):
+    """Build an Ethernet frame carrying IPv6, as the DPKT engine hands it over: its record octets attached."""
+    import dpkt
 
+    from pcapkit.toolkit import dpkt as toolkit
 
-class FakeDPKTPacket:
-    __hdr_fields__ = ('link',)
-    link = 1
+    octets = (b'\xbb' * 6 + b'\xaa' * 6 + b'\x86\xdd'
+              + struct.pack('>IHBB16s16s', 0x60000000 | flow, len(payload), nxt, 64,
+                            ip_address(src).packed, ip_address(dst).packed) + payload)
+    packet = dpkt.ethernet.Ethernet(octets)
+    toolkit.attach_buffer(packet, octets)
+    return packet
 
-    def __init__(self) -> None:
-        self.ip6 = FakeTCPIPv6()
-        self.data = self.ip6
 
-    def pack(self) -> bytes:
-        return b'packet'
+def _fake_fragment_frame():
+    """A later IPv6 fragment, behind a Hop-by-Hop header, with the fields :class:`FakeFragment` pins."""
+    frag = FakeFragment()
+    payload = (bytes([44, 0, 1, 4, 0, 0, 0, 0])                            # Hop-by-Hop, one PadN
+               + struct.pack('>BBHI', frag.nxt, 0, frag.frag_off << 3 | frag.m_flag, frag.id)
+               + b'PAYLOAD')
+    return _ipv6_frame(payload, 0, src='2001:db8::1', dst='2001:db8::2', flow=FakeIPv6.flow)
 
 
 @unittest.skipUnless(HAS_RUNTIME and HAS_DPKT, 'runtime dependencies not installed')
@@ -233,8 +218,7 @@ def test_every_adapter_hands_on_a_float_timestamp(self) -> None:
             results = {
                 'ipv4': toolkit.ipv4_reassembly(self._make_ipv4_tcp_packet(fragmented=True),
                                                 stamp, count=1),
-                'ipv6': toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=FakeIPv6(FakeFragment())),
-                                                stamp, count=1),
+                'ipv6': toolkit.ipv6_reassembly(_fake_fragment_frame(), stamp, count=1),
                 'tcp': toolkit.tcp_reassembly(packet, stamp, count=1),
                 'trace': toolkit.tcp_traceflow(packet, stamp, data_link=LinkType.ETHERNET, count=1),
             }
@@ -294,11 +278,13 @@ def fields(data):  # type: ignore[no-untyped-def]
                          (99, True, bytes(20)))
         self.assertIsNone(toolkit.ipv4_reassembly(ipv6, 1.0, count=1))
 
-    def test_tcp_helpers_cover_ipv6_and_data_payload_fallback(self) -> None:
+    def test_tcp_helpers_cover_tcp_over_ipv6(self) -> None:
         from pcapkit.const.reg.linktype import LinkType
         from pcapkit.toolkit import dpkt as toolkit
 
-        packet = FakeDPKTPacket()
+        # SYN and FIN set, with ACK
+        segment = struct.pack('>HHIIBBHHH', 2345, 443, 20, 15, 5 << 4, 0b00010011, 1024, 0, 0) + b'ip6'
+        packet = _ipv6_frame(segment, 6, src='2001:db8::10', dst='2001:db8::20')
         tcp = toolkit.tcp_reassembly(packet, 60.5, count=12)
         self.assertIsNotNone(tcp)
         assert tcp is not None
@@ -316,17 +302,18 @@ def test_tcp_helpers_cover_ipv6_and_data_payload_fallback(self) -> None:
         self.assertEqual(flow.dst, ip_address('2001:db8::20'))
         self.assertTrue(flow.syn)
         self.assertTrue(flow.fin)
-        self.assertEqual(flow.frame['ETHERNET']['FakeTCPIPv6']['TCP']['sport'], 2345)
+        self.assertEqual(flow.frame['ETHERNET']['IP6']['TCP']['sport'], 2345)
 
-    def test_ipv6_header_length_and_reassembly_with_fragment_fake(self) -> None:
+    def test_ipv6_header_length_and_reassembly_of_a_fragment(self) -> None:
         from pcapkit.const.reg.transtype import TransType
         from pcapkit.toolkit import dpkt as toolkit
 
         frag = FakeFragment()
         ipv6 = FakeIPv6(frag)
         self.assertEqual(toolkit.ipv6_hdr_len(ipv6), 48)
 
-        packet = types.SimpleNamespace(ip6=ipv6)
+        # the same fields on the wire, as the engine hands them over
+        packet = _fake_fragment_frame()
         reassembled = toolkit.ipv6_reassembly(packet, 0.0, count=5)
         self.assertIsNotNone(reassembled)
         assert reassembled is not None
@@ -350,6 +337,8 @@ def test_ipv6_header_length_and_reassembly_with_fragment_fake(self) -> None:
         self.assertIsNone(toolkit.ipv6_reassembly(types.SimpleNamespace(), 0.0, count=1))
         self.assertIsNone(toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=FakeIPv6()), 0.0,
                                                   count=1))
+        self.assertIsNone(toolkit.ipv6_reassembly(_ipv6_frame(b'PAYLOAD', 253, src='2001:db8::1',
+                                                              dst='2001:db8::2'), 0.0, count=1))
 
 
 # ---------------------------------------------------------------------------

tests/toolkit/test_tcp_carrier_walk_1606_1617_unit.py

@@ -274,18 +274,18 @@ def test_a_dpkt_packet_built_by_hand_is_walked_too(self) -> None:
     def test_a_tunnel_dpkt_cannot_decode_ends_no_extraction(self) -> None:
         # dpkt 1.9.8 reads ``frag_off`` from whatever header follows an IPv6
         # Fragment header (#1351), so the IPv6 packet behind this Hop-by-Hop
-        # header raises AttributeError. The tunnel is left undecoded, as the
-        # engine leaves a frame it cannot decode, and the next frame is read.
-        # Whether TCP is read in it, as the default engine does, is #1626.
+        # header raises AttributeError. The next frame is still read, and the
+        # TCP in this one is read off the octets, as the default engine reads
+        # it (#1626).
         if not has('dpkt'):
             self.skipTest('dpkt not installed')
-        from pcapkit.toolkit import dpkt as toolkit
+        import dpkt
 
         inner = v6(fragment(0) + options(6) + tcp(1), 44)
-        self.assertIsNone(toolkit._decode_ip(41, inner))  # pylint: disable=protected-access
+        with self.assertRaises(AttributeError):
+            dpkt.ip6.IP6(inner)
         seen = read(self, 'dpkt', [eth(v4(options(41) + inner, 0)), eth(v4(tcp(2), 6))])
-        self.assertIn(2, seen['tcp'])
-        self.assertIn(2, seen['trace'])
+        self.assertEqual((sorted(seen['tcp']), sorted(seen['trace'])), ([1, 2], [1, 2]))
 
     def test_a_dpkt_payload_that_is_no_packet_is_leaf_data(self) -> None:
         # dpkt decodes IPv4 protocol 0 as IPv4, so this Hop-by-Hop header and the