fix(engines): read each record whole through the scapy engine (#1717)

pcapkit/foundation/engines/scapy.py

@@ -63,6 +63,15 @@
 #: :func:`_reader_type` overrides.
 _READER_HOOKS = ('_read_block', '_read_block_shb', '_read_block_idb', '_check_interface_id')
 
+#: The most octets a record can hold, as its captured length is 32 bits in a
+#: PCAP record header and in every PCAP-NG packet block. As the ``size`` a
+#: packet is read with, it cuts nothing (#1717).
+_CAPLEN_MAX = 0xFFFF_FFFF
+
+#: The most octets of a PCAP record read in one call, so that a forged captured
+#: length costs only the octets the file holds, as in Scapy 2.8 (#1717).
+_READ_CHUNK = 0x40000
+
 
 def _read_tsoffset(options: 'bytes', endian: 'str') -> 'int':
     """Read the ``if_tsoffset`` of a PCAP-NG Interface Description Block (#1549).
@@ -118,6 +127,15 @@ def _reader_type() -> 'Type[PcapReader]':
     ``if_tsoffset``, the seconds to add to each of its timestamps, so the
     subclass reads it and adds it (#1549).
 
+    Both subclasses read each record whole (#1717). ``sniff`` reads a packet
+    with ``recv()``, whose ``size`` defaults to `Scapy`_'s ``MTU``, 65,535
+    octets, and every reader cuts the record to ``size``, so ``recv`` defaults
+    to :data:`_CAPLEN_MAX` instead, as does a PCAP-NG block, which `Scapy`_ 2.6
+    reads at ``MTU`` whatever the ``size``. `Scapy`_ 2.8's PCAP reader also
+    reads at most ``MTU * 4`` octets of a record, 262,140, and skips the rest,
+    so the PCAP subclass reads the record itself, :data:`_READ_CHUNK` octets at
+    a time.
+
     Built when called, so that importing this module does not import `Scapy`_,
     and so that it always subclasses the :class:`scapy.utils.PcapReader` of the
     moment.
@@ -132,7 +150,8 @@ def _reader_type() -> 'Type[PcapReader]':
     .. _Scapy: https://scapy.net
 
     """
-    from scapy.utils import EDecimal, PcapNgReader, PcapReader  # isort:skip
+    from scapy.error import warning  # isort:skip
+    from scapy.utils import EDecimal, PcapNgReader, PcapReader, RawPcapReader  # isort:skip
 
     from pcapkit.toolkit.scapy import attach_linktype, attach_resolution  # isort:skip
 
@@ -154,14 +173,44 @@ def close(self) -> 'None':
                 self._closed = True
                 super().close()  # type: ignore[misc]
 
-    class _PcapReader(_Closing, PcapReader):
+    class _Uncapped(PcapReader):
+        # ``sniff`` calls ``recv()``, and Scapy's default ``size`` of ``MTU``
+        # cut every record to 65,535 octets (#1717); an explicit one is kept
+        def recv(self, size: 'int' = _CAPLEN_MAX, **kwargs: 'Any') -> 'ScapyPacket':  # type: ignore[override]
+            return super().recv(size, **kwargs)
+
+    class _PcapRecords(RawPcapReader):
+        # Scapy 2.8 reads at most ``MTU * 4`` octets of a record and skips the
+        # rest (#1717), so the record is read here: whole, but in chunks as 2.8
+        # reads it. ``PcapReader.read_packet`` calls ``super(PcapReader,
+        # self)._read_packet``, which looks in the classes after ``PcapReader``
+        # in the reader's MRO, so this one is a base listed after it.
+        def _read_packet(self, size: 'int' = _CAPLEN_MAX) -> 'tuple[bytes, Any]':
+            header = self.f.read(16)
+            if len(header) < 16:
+                raise EOFError
+            sec, usec, caplen, wirelen = struct.unpack(f'{self.endian}IIII', header)
+            chunks = []  # type: list[bytes]
+            remaining = caplen
+            while remaining:
+                chunk = self.f.read(min(remaining, _READ_CHUNK))
+                if not chunk:
+                    # the file ends inside the record, which Scapy 2.8 warns of
+                    warning('Pcap: packet has been truncated')
+                    break
+                chunks.append(chunk)
+                remaining -= len(chunk)
+            return (b''.join(chunks)[:size],
+                    self.PacketMetadata(sec=sec, usec=usec, wirelen=wirelen, caplen=caplen))
+
+    class _PcapReader(_Closing, _Uncapped, PcapReader, _PcapRecords):
         def read_packet(self, *args: 'Any', **kwargs: 'Any') -> 'ScapyPacket':
             packet = super().read_packet(*args, **kwargs)
             attach_resolution(packet, 1_000_000_000 if self.nano else 1_000_000)
             attach_linktype(packet, self.linktype)
             return packet
 
-    class _PcapNgReader(_Closing, PcapNgReader):
+    class _PcapNgReader(_Closing, _Uncapped, PcapNgReader):
         # the metaclass makes this ``_PcapReader``'s alternative in turn
         alternative = _PcapReader
 
@@ -195,8 +244,10 @@ def _check_interface_id(self, intid: 'int') -> 'Any':
             self._interface = intid
             return super()._check_interface_id(intid)
 
-        def _read_block(self, *args: 'Any', **kwargs: 'Any') -> 'Any':
-            block = super()._read_block(*args, **kwargs)
+        def _read_block(self, size: 'int' = _CAPLEN_MAX) -> 'Any':
+            # Scapy 2.6 reads each block with ``_read_block()``, so at ``MTU``
+            # whatever the packet's ``size`` (#1717)
+            block = super()._read_block(size)
             if block is not None:
                 self._resolution = block[1].tsresol
                 self._linktype = block[1].linktype

tests/foundation/engines/test_scapy_record_length_1717_runtime.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""The scapy engine reads :file:`options-ipv6.pcap`'s jumbogram whole (#1717).
+
+:file:`examples/generators/options.py` writes HOPOPT's Jumbo Payload case as a
+genuine jumbogram, a 65,590-octet frame in a capture whose snaplen is 262,144.
+The scapy engine handed over its first 65,535 octets, Scapy's ``MTU``, which
+``recv()`` cuts every record to by default.
+
+:file:`options-ipv6.pcap` is generated, so the module belongs to the
+fixture-dependent tier.
+
+"""
+from __future__ import annotations
+
+import importlib.util
+import struct
+import unittest
+import warnings
+
+from tests._support import close_extractor, reimport_once_per_class, sample_path
+
+HAS_SCAPY = importlib.util.find_spec('scapy') is not None
+HAS_RUNTIME = all(importlib.util.find_spec(name) is not None
+                  for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper'))
+
+
+def _records(fin: 'str') -> 'list[tuple[bytes, int]]':
+    """``(octets, orig_len)`` of each record of a little-endian PCAP."""
+    with open(fin, 'rb') as file:
+        rest = file.read()[24:]
+    records = []
+    while rest:
+        _, _, incl_len, orig_len = struct.unpack('<IIII', rest[:16])
+        records.append((rest[16:16 + incl_len], orig_len))
+        rest = rest[16 + incl_len:]
+    return records
+
+
+@unittest.skipUnless(HAS_RUNTIME and HAS_SCAPY, 'runtime dependencies or scapy not installed')
+class ScapyJumbogramTests(unittest.TestCase):
+    """The scapy engine's frames hold every octet of :file:`options-ipv6.pcap`."""
+
+    def setUp(self) -> None:
+        reimport_once_per_class(self)
+
+    def test_the_jumbogram_is_read_whole(self) -> None:
+        from scapy.packet import Packet
+
+        from pcapkit import extract
+        from pcapkit.toolkit.scapy import packet2frame
+
+        fin = sample_path('options-ipv6.pcap')
+        records = _records(fin)
+        self.assertEqual([(number, len(octets)) for number, (octets, _) in enumerate(records, 1)
+                          if len(octets) > 0xFFFF], [(13, 65_590)])
+
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            extractor = extract(fin=fin, engine='scapy', nofile=True, store=True)
+        self.addCleanup(close_extractor, extractor)
+        packets = []  # type: list[Packet]
+        for packet in extractor.frame:
+            if not isinstance(packet, Packet):
+                self.fail(f'not a Scapy packet: {packet!r}')
+            packets.append(packet)
+        self.assertEqual([(bytes(packet.original), packet.wirelen) for packet in packets], records)
+
+        # and so does the PCAP frame record the flow tracer writes
+        record = packet2frame(packets[12])
+        self.assertEqual((record.frame_info.incl_len, record.frame_info.orig_len), (65_590, 65_590))
+        self.assertEqual(record.packet, records[12][0])
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/foundation/engines/test_scapy_record_length_1717_unit.py

@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+"""The scapy engine hands over each record whole (#1717).
+
+``sniff`` reads a packet with ``recv()``, whose ``size`` defaults to Scapy's
+``MTU`` of 65,535 octets, and every Scapy reader cuts a record to ``size`` --
+Scapy 2.6's PCAP-NG reader to ``MTU`` whatever the ``size``. Scapy 2.8's PCAP
+reader also reads at most ``MTU * 4`` octets of a record, 262,140, and skips
+the rest. So a record longer than either reached the engine's frames short.
+The reader subclass of :mod:`pcapkit.foundation.engines.scapy` reads it whole.
+
+The captures are synthesised, so the module belongs to the unit tier.
+
+"""
+from __future__ import annotations
+
+import importlib.util
+import os
+import struct
+import tempfile
+import unittest
+import warnings
+
+from tests._support import close_extractor, reimport_once_per_class
+
+HAS_SCAPY = importlib.util.find_spec('scapy') is not None
+HAS_RUNTIME = all(importlib.util.find_spec(name) is not None
+                  for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper'))
+
+#: ``LINKTYPE_ETHERNET``.
+ETHERNET = 1
+
+#: Record lengths: under ``MTU``; one over it; libpcap's ``MAXIMUM_SNAPLEN``,
+#: over Scapy 2.8's ``MTU * 4``; and one read in more than one chunk.
+LENGTHS = (60, 65_536, 262_144, 600_000)
+
+
+def _frame(length: 'int') -> 'bytes':
+    """An Ethernet frame of ``length`` octets, of an EtherType Scapy leaves raw."""
+    header = b'\x02' + bytes(5) + b'\x02' + bytes(4) + b'\x01' + b'\x88\xb5'
+    body = bytes(range(256)) * (length // 256 + 1)
+    return header + body[:length - len(header)]
+
+
+def _pcap(order: 'str', records: 'list[tuple[bytes, int]]', *, nanosecond: 'bool' = False) -> 'bytes':
+    """A PCAP of ``(octets, orig_len)`` records, with a snaplen over all of them."""
+    magic = 0xA1B23C4D if nanosecond else 0xA1B2C3D4
+    out = [struct.pack(f'{order}IHHiIII', magic, 2, 4, 0, 0, 0x100000, ETHERNET)]
+    for number, (octets, orig_len) in enumerate(records, 1):
+        out.append(struct.pack(f'{order}IIII', 1_500_000_000 + number, number, len(octets), orig_len) + octets)
+    return b''.join(out)
+
+
+def _block(order: 'str', kind: 'int', body: 'bytes') -> 'bytes':
+    body += bytes(-len(body) % 4)
+    length = len(body) + 12
+    return struct.pack(f'{order}II', kind, length) + body + struct.pack(f'{order}I', length)
+
+
+def _pcapng(order: 'str', octets: 'bytes') -> 'bytes':
+    """A PCAP-NG holding ``octets`` as an Enhanced, an obsolete and a Simple Packet Block."""
+    caplen = len(octets)
+    return b''.join([
+        _block(order, 0x0A0D0D0A, struct.pack(f'{order}IHHq', 0x1A2B3C4D, 1, 0, -1)),
+        _block(order, 1, struct.pack(f'{order}HHI', ETHERNET, 0, 0x100000)),
+        _block(order, 6, struct.pack(f'{order}IIIII', 0, 0, 1, caplen, caplen) + octets),
+        _block(order, 2, struct.pack(f'{order}HHIIII', 0, 0, 0, 2, caplen, caplen) + octets),
+        _block(order, 3, struct.pack(f'{order}I', caplen) + octets),
+    ])
+
+
+@unittest.skipUnless(HAS_RUNTIME and HAS_SCAPY, 'runtime dependencies or scapy not installed')
+class ScapyRecordLengthTests(unittest.TestCase):
+    """Each frame of the scapy engine holds every octet of its record."""
+
+    def setUp(self) -> None:
+        reimport_once_per_class(self)
+        tmp = tempfile.TemporaryDirectory()  # pylint: disable=consider-using-with
+        self.addCleanup(tmp.cleanup)
+        self.tmp = tmp.name
+
+    def _write(self, name: 'str', data: 'bytes') -> 'str':
+        path = os.path.join(self.tmp, name)
+        with open(path, 'wb') as file:
+            file.write(data)
+        return path
+
+    def _frames(self, fin: 'str') -> 'list[tuple[bytes, int | None]]':
+        """The octets and the original length of each frame of the scapy engine."""
+        from scapy.packet import Packet
+
+        from pcapkit import extract
+
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            extractor = extract(fin=fin, engine='scapy', nofile=True, store=True)
+        self.addCleanup(close_extractor, extractor)
+        frames = []  # type: list[tuple[bytes, int | None]]
+        for packet in extractor.frame:
+            if not isinstance(packet, Packet):
+                self.fail(f'not a Scapy packet: {packet!r}')
+            frames.append((bytes(packet.original), packet.wirelen))
+        return frames
+
+    def test_a_pcap_record_is_read_whole(self) -> None:
+        records = [(_frame(length), length) for length in LENGTHS]
+        for order in ('<', '>'):
+            for nanosecond in (False, True):
+                with self.subTest(order=order, nanosecond=nanosecond):
+                    fin = self._write(f'whole-{ord(order)}-{nanosecond}.pcap',
+                                      _pcap(order, records, nanosecond=nanosecond))
+                    frames = self._frames(fin)
+                    self.assertEqual([len(octets) for octets, _ in frames], list(LENGTHS))
+                    self.assertEqual(frames, records)
+
+    def test_a_record_the_file_ends_inside_keeps_what_it_holds(self) -> None:
+        # declared 300,000 octets long, of which the file holds 270,000
+        octets = _frame(300_000)
+        data = _pcap('<', [(_frame(60), 60), (octets, 300_000)])
+        fin = self._write('cut.pcap', data[:-30_000])
+        self.assertEqual(self._frames(fin), [(_frame(60), 60), (octets[:270_000], 300_000)])
+
+    def test_a_pcapng_packet_block_is_read_whole(self) -> None:
+        octets = _frame(70_000)
+        for order in ('<', '>'):
+            with self.subTest(order=order):
+                fin = self._write(f'whole-{ord(order)}.pcapng', _pcapng(order, octets))
+                self.assertEqual(self._frames(fin), [(octets, 70_000)] * 3)
+
+    def test_an_explicit_size_still_cuts_a_pcap_record(self) -> None:
+        # the PCAP reader's own; Scapy 2.6 reads a PCAP-NG block at ``MTU``,
+        # and so the engine's at its default, whatever ``size`` is passed
+        from pcapkit.foundation.engines.scapy import _reader_type
+
+        octets = _frame(70_000)
+        reader = _reader_type()(self._write('size.pcap', _pcap('<', [(octets, 70_000)] * 2)))
+        self.addCleanup(reader.close)
+        self.assertEqual(bytes(reader.recv(100).original), octets[:100])
+        self.assertEqual(bytes(reader.recv().original), octets)
+
+
+if __name__ == '__main__':
+    unittest.main()