From 910d103453a0cad0f7c72dc6fc35c210db4e764d Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 14 Oct 2026 12:09:22 -0400 Subject: [PATCH] fix(engines): read each record whole through the scapy engine (#1717) - sniff() reads with recv(), whose size defaults to Scapy's MTU of 65,535 octets, and every Scapy reader cuts a record to size: the engine's readers default recv() to 0xFFFFFFFF, the most a 32-bit captured length counts, for PCAP and PCAP-NG alike; an explicit size is still passed on - Scapy 2.6 reads each PCAP-NG block at MTU whatever the size: the _read_block() override defaults to the same 0xFFFFFFFF - Scapy 2.8's PCAP reader reads at most MTU * 4 (262,140) octets of a record and skips the rest: the PCAP reader reads the record itself, in 256 KiB chunks so a forged captured length still costs only the octets the file holds, from a RawPcapReader base after PcapReader, where PcapReader.read_packet's super(PcapReader, self) looks Tests: test_scapy_record_length_1717_{unit,runtime}.py, 5 passed on scapy 2.6.0, 2.7.0 and 2.8.0; 9 of the 11 fail on each against origin/main. Closes #1717 --- pcapkit/foundation/engines/scapy.py | 61 +++++++- .../test_scapy_record_length_1717_runtime.py | 75 +++++++++ .../test_scapy_record_length_1717_unit.py | 142 ++++++++++++++++++ 3 files changed, 273 insertions(+), 5 deletions(-) create mode 100644 tests/foundation/engines/test_scapy_record_length_1717_runtime.py create mode 100644 tests/foundation/engines/test_scapy_record_length_1717_unit.py diff --git a/pcapkit/foundation/engines/scapy.py b/pcapkit/foundation/engines/scapy.py index fd61b4ea02..ad3776674f 100644 --- a/pcapkit/foundation/engines/scapy.py +++ b/pcapkit/foundation/engines/scapy.py @@ -63,6 +63,15 @@ #: :func:`_reader_type` overrides. _READER_HOOKS = ('_read_block', '_read_block_shb', '_read_block_idb', '_check_interface_id') +#: The most octets a record can hold, as its captured length is 32 bits in a +#: PCAP record header and in every PCAP-NG packet block. As the ``size`` a +#: packet is read with, it cuts nothing (#1717). +_CAPLEN_MAX = 0xFFFF_FFFF + +#: The most octets of a PCAP record read in one call, so that a forged captured +#: length costs only the octets the file holds, as in Scapy 2.8 (#1717). +_READ_CHUNK = 0x40000 + def _read_tsoffset(options: 'bytes', endian: 'str') -> 'int': """Read the ``if_tsoffset`` of a PCAP-NG Interface Description Block (#1549). @@ -118,6 +127,15 @@ def _reader_type() -> 'Type[PcapReader]': ``if_tsoffset``, the seconds to add to each of its timestamps, so the subclass reads it and adds it (#1549). + Both subclasses read each record whole (#1717). ``sniff`` reads a packet + with ``recv()``, whose ``size`` defaults to `Scapy`_'s ``MTU``, 65,535 + octets, and every reader cuts the record to ``size``, so ``recv`` defaults + to :data:`_CAPLEN_MAX` instead, as does a PCAP-NG block, which `Scapy`_ 2.6 + reads at ``MTU`` whatever the ``size``. `Scapy`_ 2.8's PCAP reader also + reads at most ``MTU * 4`` octets of a record, 262,140, and skips the rest, + so the PCAP subclass reads the record itself, :data:`_READ_CHUNK` octets at + a time. + Built when called, so that importing this module does not import `Scapy`_, and so that it always subclasses the :class:`scapy.utils.PcapReader` of the moment. @@ -132,7 +150,8 @@ def _reader_type() -> 'Type[PcapReader]': .. _Scapy: https://scapy.net """ - from scapy.utils import EDecimal, PcapNgReader, PcapReader # isort:skip + from scapy.error import warning # isort:skip + from scapy.utils import EDecimal, PcapNgReader, PcapReader, RawPcapReader # isort:skip from pcapkit.toolkit.scapy import attach_linktype, attach_resolution # isort:skip @@ -154,14 +173,44 @@ def close(self) -> 'None': self._closed = True super().close() # type: ignore[misc] - class _PcapReader(_Closing, PcapReader): + class _Uncapped(PcapReader): + # ``sniff`` calls ``recv()``, and Scapy's default ``size`` of ``MTU`` + # cut every record to 65,535 octets (#1717); an explicit one is kept + def recv(self, size: 'int' = _CAPLEN_MAX, **kwargs: 'Any') -> 'ScapyPacket': # type: ignore[override] + return super().recv(size, **kwargs) + + class _PcapRecords(RawPcapReader): + # Scapy 2.8 reads at most ``MTU * 4`` octets of a record and skips the + # rest (#1717), so the record is read here: whole, but in chunks as 2.8 + # reads it. ``PcapReader.read_packet`` calls ``super(PcapReader, + # self)._read_packet``, which looks in the classes after ``PcapReader`` + # in the reader's MRO, so this one is a base listed after it. + def _read_packet(self, size: 'int' = _CAPLEN_MAX) -> 'tuple[bytes, Any]': + header = self.f.read(16) + if len(header) < 16: + raise EOFError + sec, usec, caplen, wirelen = struct.unpack(f'{self.endian}IIII', header) + chunks = [] # type: list[bytes] + remaining = caplen + while remaining: + chunk = self.f.read(min(remaining, _READ_CHUNK)) + if not chunk: + # the file ends inside the record, which Scapy 2.8 warns of + warning('Pcap: packet has been truncated') + break + chunks.append(chunk) + remaining -= len(chunk) + return (b''.join(chunks)[:size], + self.PacketMetadata(sec=sec, usec=usec, wirelen=wirelen, caplen=caplen)) + + class _PcapReader(_Closing, _Uncapped, PcapReader, _PcapRecords): def read_packet(self, *args: 'Any', **kwargs: 'Any') -> 'ScapyPacket': packet = super().read_packet(*args, **kwargs) attach_resolution(packet, 1_000_000_000 if self.nano else 1_000_000) attach_linktype(packet, self.linktype) return packet - class _PcapNgReader(_Closing, PcapNgReader): + class _PcapNgReader(_Closing, _Uncapped, PcapNgReader): # the metaclass makes this ``_PcapReader``'s alternative in turn alternative = _PcapReader @@ -195,8 +244,10 @@ def _check_interface_id(self, intid: 'int') -> 'Any': self._interface = intid return super()._check_interface_id(intid) - def _read_block(self, *args: 'Any', **kwargs: 'Any') -> 'Any': - block = super()._read_block(*args, **kwargs) + def _read_block(self, size: 'int' = _CAPLEN_MAX) -> 'Any': + # Scapy 2.6 reads each block with ``_read_block()``, so at ``MTU`` + # whatever the packet's ``size`` (#1717) + block = super()._read_block(size) if block is not None: self._resolution = block[1].tsresol self._linktype = block[1].linktype diff --git a/tests/foundation/engines/test_scapy_record_length_1717_runtime.py b/tests/foundation/engines/test_scapy_record_length_1717_runtime.py new file mode 100644 index 0000000000..5a5cd4e3dc --- /dev/null +++ b/tests/foundation/engines/test_scapy_record_length_1717_runtime.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +"""The scapy engine reads :file:`options-ipv6.pcap`'s jumbogram whole (#1717). + +:file:`examples/generators/options.py` writes HOPOPT's Jumbo Payload case as a +genuine jumbogram, a 65,590-octet frame in a capture whose snaplen is 262,144. +The scapy engine handed over its first 65,535 octets, Scapy's ``MTU``, which +``recv()`` cuts every record to by default. + +:file:`options-ipv6.pcap` is generated, so the module belongs to the +fixture-dependent tier. + +""" +from __future__ import annotations + +import importlib.util +import struct +import unittest +import warnings + +from tests._support import close_extractor, reimport_once_per_class, sample_path + +HAS_SCAPY = importlib.util.find_spec('scapy') is not None +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None + for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) + + +def _records(fin: 'str') -> 'list[tuple[bytes, int]]': + """``(octets, orig_len)`` of each record of a little-endian PCAP.""" + with open(fin, 'rb') as file: + rest = file.read()[24:] + records = [] + while rest: + _, _, incl_len, orig_len = struct.unpack(' None: + reimport_once_per_class(self) + + def test_the_jumbogram_is_read_whole(self) -> None: + from scapy.packet import Packet + + from pcapkit import extract + from pcapkit.toolkit.scapy import packet2frame + + fin = sample_path('options-ipv6.pcap') + records = _records(fin) + self.assertEqual([(number, len(octets)) for number, (octets, _) in enumerate(records, 1) + if len(octets) > 0xFFFF], [(13, 65_590)]) + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + extractor = extract(fin=fin, engine='scapy', nofile=True, store=True) + self.addCleanup(close_extractor, extractor) + packets = [] # type: list[Packet] + for packet in extractor.frame: + if not isinstance(packet, Packet): + self.fail(f'not a Scapy packet: {packet!r}') + packets.append(packet) + self.assertEqual([(bytes(packet.original), packet.wirelen) for packet in packets], records) + + # and so does the PCAP frame record the flow tracer writes + record = packet2frame(packets[12]) + self.assertEqual((record.frame_info.incl_len, record.frame_info.orig_len), (65_590, 65_590)) + self.assertEqual(record.packet, records[12][0]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/foundation/engines/test_scapy_record_length_1717_unit.py b/tests/foundation/engines/test_scapy_record_length_1717_unit.py new file mode 100644 index 0000000000..6df7769c95 --- /dev/null +++ b/tests/foundation/engines/test_scapy_record_length_1717_unit.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +"""The scapy engine hands over each record whole (#1717). + +``sniff`` reads a packet with ``recv()``, whose ``size`` defaults to Scapy's +``MTU`` of 65,535 octets, and every Scapy reader cuts a record to ``size`` -- +Scapy 2.6's PCAP-NG reader to ``MTU`` whatever the ``size``. Scapy 2.8's PCAP +reader also reads at most ``MTU * 4`` octets of a record, 262,140, and skips +the rest. So a record longer than either reached the engine's frames short. +The reader subclass of :mod:`pcapkit.foundation.engines.scapy` reads it whole. + +The captures are synthesised, so the module belongs to the unit tier. + +""" +from __future__ import annotations + +import importlib.util +import os +import struct +import tempfile +import unittest +import warnings + +from tests._support import close_extractor, reimport_once_per_class + +HAS_SCAPY = importlib.util.find_spec('scapy') is not None +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None + for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) + +#: ``LINKTYPE_ETHERNET``. +ETHERNET = 1 + +#: Record lengths: under ``MTU``; one over it; libpcap's ``MAXIMUM_SNAPLEN``, +#: over Scapy 2.8's ``MTU * 4``; and one read in more than one chunk. +LENGTHS = (60, 65_536, 262_144, 600_000) + + +def _frame(length: 'int') -> 'bytes': + """An Ethernet frame of ``length`` octets, of an EtherType Scapy leaves raw.""" + header = b'\x02' + bytes(5) + b'\x02' + bytes(4) + b'\x01' + b'\x88\xb5' + body = bytes(range(256)) * (length // 256 + 1) + return header + body[:length - len(header)] + + +def _pcap(order: 'str', records: 'list[tuple[bytes, int]]', *, nanosecond: 'bool' = False) -> 'bytes': + """A PCAP of ``(octets, orig_len)`` records, with a snaplen over all of them.""" + magic = 0xA1B23C4D if nanosecond else 0xA1B2C3D4 + out = [struct.pack(f'{order}IHHiIII', magic, 2, 4, 0, 0, 0x100000, ETHERNET)] + for number, (octets, orig_len) in enumerate(records, 1): + out.append(struct.pack(f'{order}IIII', 1_500_000_000 + number, number, len(octets), orig_len) + octets) + return b''.join(out) + + +def _block(order: 'str', kind: 'int', body: 'bytes') -> 'bytes': + body += bytes(-len(body) % 4) + length = len(body) + 12 + return struct.pack(f'{order}II', kind, length) + body + struct.pack(f'{order}I', length) + + +def _pcapng(order: 'str', octets: 'bytes') -> 'bytes': + """A PCAP-NG holding ``octets`` as an Enhanced, an obsolete and a Simple Packet Block.""" + caplen = len(octets) + return b''.join([ + _block(order, 0x0A0D0D0A, struct.pack(f'{order}IHHq', 0x1A2B3C4D, 1, 0, -1)), + _block(order, 1, struct.pack(f'{order}HHI', ETHERNET, 0, 0x100000)), + _block(order, 6, struct.pack(f'{order}IIIII', 0, 0, 1, caplen, caplen) + octets), + _block(order, 2, struct.pack(f'{order}HHIIII', 0, 0, 0, 2, caplen, caplen) + octets), + _block(order, 3, struct.pack(f'{order}I', caplen) + octets), + ]) + + +@unittest.skipUnless(HAS_RUNTIME and HAS_SCAPY, 'runtime dependencies or scapy not installed') +class ScapyRecordLengthTests(unittest.TestCase): + """Each frame of the scapy engine holds every octet of its record.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + + def _write(self, name: 'str', data: 'bytes') -> 'str': + path = os.path.join(self.tmp, name) + with open(path, 'wb') as file: + file.write(data) + return path + + def _frames(self, fin: 'str') -> 'list[tuple[bytes, int | None]]': + """The octets and the original length of each frame of the scapy engine.""" + from scapy.packet import Packet + + from pcapkit import extract + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + extractor = extract(fin=fin, engine='scapy', nofile=True, store=True) + self.addCleanup(close_extractor, extractor) + frames = [] # type: list[tuple[bytes, int | None]] + for packet in extractor.frame: + if not isinstance(packet, Packet): + self.fail(f'not a Scapy packet: {packet!r}') + frames.append((bytes(packet.original), packet.wirelen)) + return frames + + def test_a_pcap_record_is_read_whole(self) -> None: + records = [(_frame(length), length) for length in LENGTHS] + for order in ('<', '>'): + for nanosecond in (False, True): + with self.subTest(order=order, nanosecond=nanosecond): + fin = self._write(f'whole-{ord(order)}-{nanosecond}.pcap', + _pcap(order, records, nanosecond=nanosecond)) + frames = self._frames(fin) + self.assertEqual([len(octets) for octets, _ in frames], list(LENGTHS)) + self.assertEqual(frames, records) + + def test_a_record_the_file_ends_inside_keeps_what_it_holds(self) -> None: + # declared 300,000 octets long, of which the file holds 270,000 + octets = _frame(300_000) + data = _pcap('<', [(_frame(60), 60), (octets, 300_000)]) + fin = self._write('cut.pcap', data[:-30_000]) + self.assertEqual(self._frames(fin), [(_frame(60), 60), (octets[:270_000], 300_000)]) + + def test_a_pcapng_packet_block_is_read_whole(self) -> None: + octets = _frame(70_000) + for order in ('<', '>'): + with self.subTest(order=order): + fin = self._write(f'whole-{ord(order)}.pcapng', _pcapng(order, octets)) + self.assertEqual(self._frames(fin), [(octets, 70_000)] * 3) + + def test_an_explicit_size_still_cuts_a_pcap_record(self) -> None: + # the PCAP reader's own; Scapy 2.6 reads a PCAP-NG block at ``MTU``, + # and so the engine's at its default, whatever ``size`` is passed + from pcapkit.foundation.engines.scapy import _reader_type + + octets = _frame(70_000) + reader = _reader_type()(self._write('size.pcap', _pcap('<', [(octets, 70_000)] * 2))) + self.addCleanup(reader.close) + self.assertEqual(bytes(reader.recv(100).original), octets[:100]) + self.assertEqual(bytes(reader.recv().original), octets) + + +if __name__ == '__main__': + unittest.main()