From b4702f17c2fdb4938ad5858e58146ae24aa443fb Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 9 Oct 2026 22:10:20 -0400 Subject: [PATCH] fix(engines): dpkt float timestamps, Simple Packet Blocks and raw-IP tracing (#1501, #1520, #1548) - The dpkt reassembly and flow-tracing adapters passed the reader's timestamp straight through, so a nanosecond PCAP gave reassembly and the flow tracer a Decimal where every other engine gives a float, and flow labels were spelled with nine places. They now hand on float(timestamp), as the default and scapy toolkits do; the PCAP frame record is still built from the exact timestamp attached to the frame, so traces stay byte-exact. - PCAPNGReader skipped Simple Packet Blocks, dropping their packets and renumbering every later frame. It now parses an SPB with pcapkit's own PCAPNG against interface 0, dates it at the epoch as the default engine does, and keeps the block's original length. The engine docs no longer say that dpkt drops them. - On a raw-IP link type (228, 229) dpkt reads a frame as a bare IP or IP6, which names no ip or ip6 layer, so all four adapters returned None and nothing was reassembled or traced. They now take such a packet as its own network layer. - Deletes the Gap(1501) and Gap(1520) rows from the engine-agreement table, and the labels=False workaround from the nanosecond PCAP trace test. --- .../pcapkit/foundation/engines/3rdparty.rst | 26 +--- pcapkit/foundation/engines/dpkt.py | 58 ++++++++- pcapkit/toolkit/dpkt.py | 69 +++++++--- .../test_dpkt_pcapng_interfaces_unit.py | 120 +++++++++++++++++- .../engines/test_engine_agreement_runtime.py | 13 -- tests/toolkit/test_dpkt_unit.py | 78 ++++++++++++ .../toolkit/test_engine_pcap_trace_runtime.py | 28 +++- 7 files changed, 323 insertions(+), 69 deletions(-) diff --git a/docs/source/pcapkit/foundation/engines/3rdparty.rst b/docs/source/pcapkit/foundation/engines/3rdparty.rst index 2efcc71e6a..10a712e211 100644 --- a/docs/source/pcapkit/foundation/engines/3rdparty.rst +++ b/docs/source/pcapkit/foundation/engines/3rdparty.rst @@ -64,26 +64,6 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. .. _DPKT: https://dpkt.readthedocs.io -.. important:: - - On PCAP-NG, `DPKT`_ yields **Enhanced Packet Blocks and (obsolete) Packet - Blocks only**. ``dpkt.pcapng.Reader.__iter__`` skips every other block type - without a warning (``# just ignore other blocks``, as of ``dpkt`` 1.9.8), and - :class:`~pcapkit.foundation.engines.dpkt.PCAPNGReader`, which this engine - reads PCAP-NG with, keeps that block set. It includes the **Simple Packet - Block**, which carries a packet just as an Enhanced Packet Block does. Such packets are silently missing from this - engine's output, and the frames after them are numbered as if they had never - been captured. - - Reading :file:`examples/captures/test.pcapng` -- four Enhanced Packet Blocks - and one Simple Packet Block, across two sections -- this engine gives four - frames where the default engine and - :class:`~pcapkit.foundation.engines.scapy.Scapy` give five; the missing one is - frame 4, the Simple Packet Block. The second Section Header Block is not the - cause: the Enhanced Packet Block after it is read. This is a limitation of - `DPKT`_, not of :mod:`pcapkit`, so read a capture that may contain Simple - Packet Blocks with the default engine instead. - .. note:: ``dpkt.pcapng.Reader`` reads only the **first** Interface Description Block @@ -93,6 +73,12 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. resolves each packet's link type, ``if_tsresol`` and ``if_tsoffset`` from the interface it names, in its own section (:issue:`1379`). + ``dpkt.pcapng.Reader`` also skips Simple Packet Blocks, which `DPKT`_ has no + type for; :class:`~pcapkit.foundation.engines.dpkt.PCAPNGReader` reads them + through :mod:`pcapkit`'s own :class:`~pcapkit.protocols.misc.pcapng.PCAPNG` + and, since the block carries no timestamp, dates them 0, as the default + engine does (:issue:`1520`). + .. autoclass:: pcapkit.foundation.engines.dpkt.DPKT :no-members: :show-inheritance: diff --git a/pcapkit/foundation/engines/dpkt.py b/pcapkit/foundation/engines/dpkt.py index 6139fe90d7..4a05b42a79 100644 --- a/pcapkit/foundation/engines/dpkt.py +++ b/pcapkit/foundation/engines/dpkt.py @@ -32,9 +32,11 @@ from pcapkit.foundation.engines.pcapng import Context from pcapkit.foundation.extraction import Extractor + from pcapkit.protocols.data.misc.pcapng import IF_TSResolOption as Data_IF_TSResolOption from pcapkit.protocols.data.misc.pcapng import \ InterfaceDescriptionBlock as Data_InterfaceDescriptionBlock from pcapkit.protocols.data.misc.pcapng import SectionHeaderBlock as Data_SectionHeaderBlock + from pcapkit.protocols.data.misc.pcapng import SimplePacketBlock as Data_SimplePacketBlock Reader = Union[PCAPReader, 'PCAPNGReader'] @@ -55,8 +57,11 @@ class PCAPNGReader: ``if_tsresol`` and ``if_tsoffset`` from the interface it names. Packet blocks are still decoded by `DPKT`_. - As :class:`dpkt.pcapng.Reader` does, it yields Enhanced Packet Blocks and - (obsolete) Packet Blocks only, and skips every other block. + It yields Enhanced Packet Blocks, (obsolete) Packet Blocks and Simple Packet + Blocks, and skips every other block. :class:`dpkt.pcapng.Reader` skips the + Simple Packet Block too, and `DPKT`_ has no type for one, so that block is + parsed with :class:`~pcapkit.protocols.misc.pcapng.PCAPNG` as well + (:issue:`1520`). .. _DPKT: https://dpkt.readthedocs.io @@ -123,6 +128,8 @@ def _read(self) -> 'Iterator[tuple[float, bytes]]': self._ctx.interfaces.append(cast('Data_InterfaceDescriptionBlock', idb.info)) elif block_type in (Enum_BlockType.Enhanced_Packet_Block, Enum_BlockType.Packet_Block): yield self._read_packet(buf, block_type, byteorder == 'little') + elif block_type == Enum_BlockType.Simple_Packet_Block: + yield self._read_simple(buf) def _read_packet(self, buf: 'bytes', block_type: 'int', little: 'bool') -> 'tuple[float, bytes]': @@ -177,6 +184,46 @@ def _read_packet(self, buf: 'bytes', block_type: 'int', return (Timestamp(float(timestamp), timestamp, resolution, pkt.pkt_len), # pylint: disable=no-member pkt.pkt_data) + def _read_simple(self, buf: 'bytes') -> 'tuple[float, bytes]': + """Decode a Simple Packet Block, against the section's first interface. + + `DPKT`_ has no type for this block, so it is parsed by + :class:`~pcapkit.protocols.misc.pcapng.PCAPNG`, as for the default + engine: the packet data is ``min(original_len, snaplen)`` octets, with + the snaplen of interface 0. + + Args: + buf: Whole block, as read from the file. + + Returns: + Timestamp and packet data, as :meth:`_read_packet` returns them. A + Simple Packet Block carries no timestamp, so the packet is dated at + the UNIX epoch, as the default engine dates it; the original length + is the block's own. + + Raises: + FormatError: If the section describes no interface. A Simple Packet + Block names none, so it refers to the first (Section 4.4 of the + PCAP-NG specification). + + """ + from pcapkit.protocols.misc.pcapng import PCAPNG as P_PCAPNG + from pcapkit.toolkit.dpkt import Timestamp # isort:skip + + if not self._ctx.interfaces: + raise FormatError('PCAP-NG: [SPB] section has no interface description block') + interface = self._ctx.interfaces[0] + block = P_PCAPNG(buf, num=0, sct=self._sect, ctx=self._ctx) + spb = cast('Data_SimplePacketBlock', block.info) + + tsresol = cast('Optional[Data_IF_TSResolOption]', + interface.options.get(Enum_OptionType.if_tsresol)) + resolution = 1_000_000 if tsresol is None else tsresol.resolution + + self._linktype = interface.linktype + return (Timestamp(0.0, decimal.Decimal(0), resolution, spb.original_len), + bytes(spb.packet)) + def _pcap_reader() -> 'Type[PCAPReader]': """A :class:`dpkt.pcap.Reader` that keeps each record's original length. @@ -258,10 +305,9 @@ class DPKT(EngineBase['DPKTPacket']): :class:`dpkt.pcapng.Reader`, so each packet takes the link type and the timestamp resolution and offset of the interface it was captured on. - Like `DPKT`_'s reader, it yields Enhanced Packet Blocks and (obsolete) - Packet Blocks only, and skips every other block without a warning -- the - Simple Packet Block included. A packet carried in a Simple Packet Block is - therefore missing from this engine's output; the default engine reads it. + It reads every packet block the default engine reads -- the Enhanced, the + Simple and the (obsolete) Packet Block -- and skips every other block + without a warning. `DPKT`_'s own reader skips the Simple Packet Block too. .. _DPKT: https://dpkt.readthedocs.io diff --git a/pcapkit/toolkit/dpkt.py b/pcapkit/toolkit/dpkt.py index 756afd3b46..b4d2d9a29d 100644 --- a/pcapkit/toolkit/dpkt.py +++ b/pcapkit/toolkit/dpkt.py @@ -81,8 +81,9 @@ class RecordTimestamp: resolution and its original length would be lost between the reader and :func:`packet2frame`. The engine's readers yield a :class:`Timestamp` or a :class:`DecimalTimestamp` instead, which *is* the value `DPKT`_ yields -- - so reassembly and flow labels see exactly what they did before -- and also - carries those three. + so a stored frame's timestamp is exactly what it was before -- and also + carries those three. Reassembly and flow tracing are handed a plain + :obj:`float` of it, as by every other engine (:issue:`1501`). .. _DPKT: https://dpkt.readthedocs.io @@ -395,6 +396,30 @@ def _layer2bytes(packet: 'Packet', layer: 'Packet') -> 'tuple[bytes, bool]': return packet2bytes(layer), False +def _network(packet: 'Packet') -> 'Optional[IP | IP6]': + """Fetch the network layer of a DPKT packet. + + Args: + packet: DPKT packet, i.e. the outermost layer. + + Returns: + The packet itself if it is an IPv4 or IPv6 packet, as `DPKT`_ reads every + frame of a raw-IP link type (:issue:`1548`); otherwise the IPv4 or IPv6 + payload of its link layer, which `DPKT`_ names ``ip`` or ``ip6`` -- a + bare IP packet names neither -- or :data:`None` if it has none. + + .. _DPKT: https://dpkt.readthedocs.io + + """ + import dpkt # isort:skip + + if isinstance(packet, (dpkt.ip.IP, dpkt.ip6.IP6)): + return packet + if hasattr(packet, 'ip'): + return cast('IP', packet.ip) + return getattr(packet, 'ip6', None) + + def ipv6_hdr_len(ipv6: 'IP6') -> 'int': """Calculate length of headers before IPv6 Fragment header. @@ -482,7 +507,9 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, than on the packet, so it is passed in -- as :func:`tcp_traceflow` does. A caller holding only a frame can read it back with :func:`packet2timestamp`, which is where - :class:`~pcapkit.foundation.engines.dpkt.DPKT` leaves it. + :class:`~pcapkit.foundation.engines.dpkt.DPKT` leaves it. Handed on + as a :obj:`float`, as every engine's adapters do, though `DPKT`_ reads + a nanosecond PCAP's as a :class:`~decimal.Decimal` (:issue:`1501`). count: Packet index. If not provided, default to ``-1``. Returns: @@ -507,7 +534,9 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.reassembly.ipv4.IPv4` """ - ipv4 = getattr(packet, 'ip', None) # type: Optional[IP] + network = _network(packet) + # an IPv6 layer is the one with extension headers, as in ``_wire_hdr_len`` + ipv4 = None if hasattr(network, 'extension_hdrs') else network if ipv4 is not None: if ipv4.df: # dismiss not fragmented packet return None @@ -537,7 +566,7 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, tl=ipv4.len, # total length, header includes header=wire[:ihl], # raw bytes type header payload=bytearray(wire[ihl:]), # raw bytearray type payload - timestamp=timestamp, # capture timestamp + timestamp=float(timestamp), # capture timestamp ) return data return None @@ -551,6 +580,7 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, packet: DPKT packet. timestamp: Capture timestamp of the packet, which drives the reassembly timeout; :func:`packet2timestamp` reads it back off a stored frame. + Handed on as a :obj:`float`, as for :func:`ipv4_reassembly`. count: Packet index. If not provided, default to ``-1``. Returns: @@ -575,7 +605,8 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.reassembly.ipv6.IPv6` """ - ipv6 = getattr(packet, 'ip6', None) # type: Optional[IP6] + network = _network(packet) + ipv6 = cast('IP6', network) if hasattr(network, 'extension_hdrs') else None if ipv6 is not None: ipv6_frag = ipv6.extension_hdrs.get(44) # type: Optional[IP6FragmentHeader] if ipv6_frag is None: # dismiss not fragmented packet @@ -618,7 +649,7 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, tl=hdr_len + len(payload), # total length, header includes header=wire[:hdr_len], # raw bytes type header before IPv6-Frag payload=bytearray(payload), # raw bytearray type payload after IPv6-Frag - timestamp=timestamp, # capture timestamp + timestamp=float(timestamp), # capture timestamp ) return data return None @@ -632,6 +663,7 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, packet: DPKT packet. timestamp: Capture timestamp of the packet, which drives the reassembly timeout; :func:`packet2timestamp` reads it back off a stored frame. + Handed on as a :obj:`float`, as for :func:`ipv4_reassembly`. count: Packet index. If not provided, default to ``-1``. Returns: @@ -655,11 +687,8 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.reassembly.tcp.TCP` """ - if hasattr(packet, 'ip'): - ip = cast('IP', packet.ip) - elif hasattr(packet, 'ip6'): - ip = cast('IP6', packet.ip6) - else: + ip = _network(packet) + if ip is None: return None tcp = getattr(ip, 'tcp', None) # type: Optional[TCP] @@ -688,7 +717,7 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, first=tcp.seq, # first sequence number of payload last=tcp.seq + raw_len - 1, # last sequence number of payload len=raw_len, # payload length, header excludes - timestamp=timestamp, # capture timestamp + timestamp=float(timestamp), # capture timestamp ) return data return None @@ -700,7 +729,10 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, Args: packet: DPKT packet. - timestamp: Timestamp of the packet. + timestamp: Timestamp of the packet, handed on as a :obj:`float`, as for + :func:`ipv4_reassembly`. The PCAP frame record is built from the + exact timestamp attached by :func:`attach_timestamp`, if any; see + :func:`packet2frame`. data_link: Data link layer protocol (from global header). count: Packet index. If not provided, default to ``-1``. @@ -725,11 +757,8 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.traceflow.tcp.TCP` """ - if hasattr(packet, 'ip'): - ip = cast('IP', packet.ip) - elif hasattr(packet, 'ip6'): - ip = cast('IP6', packet.ip6) - else: + ip = _network(packet) + if ip is None: return None tcp = getattr(ip, 'tcp', None) # type: Optional[TCP] @@ -750,7 +779,7 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, dst=ipaddress.ip_address(ip.dst), # destination IP srcport=tcp.sport, # TCP source port dstport=tcp.dport, # TCP destination port - timestamp=timestamp, # timestamp + timestamp=float(timestamp), # timestamp seq=tcp.seq, # TCP sequence number ack=tcp.ack, # TCP acknowledgement number header=wire[:tcp.off * 4], # raw bytes type header diff --git a/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py b/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py index 1fabd22040..bc9f1e7b4d 100644 --- a/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py +++ b/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py @@ -12,6 +12,9 @@ ``if_tsoffset``. Everything from :mod:`pcapkit` is imported inside each test, after :func:`~tests._support.reimport_once_per_class`, not at module load. +GitHub issue #1520: the engine's reader skipped Simple Packet Blocks, so their +packets were missing and every later frame was numbered one short. + """ import importlib.util @@ -21,12 +24,15 @@ import unittest import warnings -from tests._support import reimport_once_per_class +from tests._support import close_extractor, reimport_once_per_class HAS_DPKT = importlib.util.find_spec('dpkt') is not None #: Minimal IPv4 header (no payload), protocol 59 -- usable under any IP link type. IPV4 = bytes.fromhex('45000014000100004000') + bytes(2) + bytes([10, 0, 0, 1, 10, 0, 0, 2]) +#: IPv4 header (protocol 6) followed by a bare TCP SYN, from port 1234 to 80. +IPV4_TCP = (bytes.fromhex('45000028000100004006') + bytes(2) + bytes([10, 0, 0, 1, 10, 0, 0, 2]) + + struct.pack('>HHIIBBHHH', 1234, 80, 1, 0, 0x50, 0x02, 8192, 0, 0)) #: Ethernet frame carrying :data:`IPV4`. ETHERNET = bytes(6) + bytes([2, 0, 0, 0, 0, 1]) + b'\x08\x00' + IPV4 @@ -49,7 +55,7 @@ def _shb(order: str) -> bytes: def _idb(order: str, linktype: int, *, tsresol: 'int | None' = None, - tsoffset: 'int | None' = None) -> bytes: + tsoffset: 'int | None' = None, snaplen: int = 65535) -> bytes: options = b'' if tsresol is not None: options += _pad(struct.pack(f'{order}HHB', 9, 1, tsresol)) @@ -57,7 +63,7 @@ def _idb(order: str, linktype: int, *, tsresol: 'int | None' = None, options += struct.pack(f'{order}HHq', 14, 8, tsoffset) if options: options += struct.pack(f'{order}HH', 0, 0) - return _block(order, 1, struct.pack(f'{order}HHI', linktype, 0, 65535) + options) + return _block(order, 1, struct.pack(f'{order}HHI', linktype, 0, snaplen) + options) def _epb(order: str, iface: int, data: bytes) -> bytes: @@ -65,6 +71,11 @@ def _epb(order: str, iface: int, data: bytes) -> bytes: len(data), len(data)) + data) +def _spb(order: str, data: bytes, original_len: 'int | None' = None) -> bytes: + original_len = len(data) if original_len is None else original_len + return _block(order, 3, struct.pack(f'{order}I', original_len) + data) + + #: Packets of :func:`_capture`: (link type, parsed-as, timestamp), in file order. EXPECTED = [ (1, 'Ethernet', TICKS / 10**6), # section 1, interface 0 @@ -153,5 +164,108 @@ def test_unknown_interface_id_is_a_format_error(self) -> None: next(PCAPNGReader(file)) + +@unittest.skipUnless(HAS_DPKT, 'dpkt is not installed') +class TestDPKTSimplePacketBlock(unittest.TestCase): + """The DPKT engine reads a Simple Packet Block, as the default engine does.""" + + #: Ethernet, then raw IPv4 on nanoseconds; an SPB names no interface, so it + #: is on the first, whose snaplen cuts it short. Big-endian, so the block's + #: own fields are read in the section's byte order. + CAPTURE = b''.join([ + _shb('>'), + _idb('>', 1, snaplen=len(ETHERNET)), + _idb('>', 101, tsresol=9), + _epb('>', 1, IPV4), + _spb('>', ETHERNET, original_len=len(ETHERNET) + 40), + _epb('>', 0, ETHERNET), + ]) + + def setUp(self) -> None: + reimport_once_per_class(self) + self._tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + self.addCleanup(self._tmp.cleanup) + self.path = os.path.join(self._tmp.name, 'simple.pcapng') + + def _write(self, data: bytes) -> str: + with open(self.path, 'wb') as file: + file.write(data) + return self.path + + def test_reader_yields_the_block_on_the_first_interface(self) -> None: + import decimal + + from pcapkit.foundation.engines.dpkt import PCAPNGReader + + with open(self._write(self.CAPTURE), 'rb') as file: + reader = PCAPNGReader(file) + seen = [(int(reader.datalink()), timestamp, data) for timestamp, data in reader] + + self.assertEqual([(lt, data) for lt, _, data in seen], + [(101, IPV4), (1, ETHERNET), (1, ETHERNET)]) + # no timestamp in the block: dated at the epoch, as the default engine + # dates it; the original length is the block's, not the captured one, + # and the data stops at the snaplen, short of the block's padding + stamp = seen[1][1] + self.assertEqual((float(stamp), stamp.exact, stamp.orig_len), + (0.0, decimal.Decimal(0), len(ETHERNET) + 40)) + + def test_dpkt_engine_matches_default_engine(self) -> None: + import pcapkit + from pcapkit.toolkit.dpkt import packet2bytes, packet2timestamp + + self._write(self.CAPTURE) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + default = pcapkit.extract(fin=self.path, engine='default', nofile=True, store=True) + dpkt = pcapkit.extract(fin=self.path, engine='dpkt', nofile=True, store=True) + + self.assertEqual(dpkt.length, default.length) + self.assertEqual([(float(packet2timestamp(frame)), packet2bytes(frame)) + for frame in dpkt.frame], + [(float(frame.info.timestamp_epoch), bytes(frame.info.packet)) + for frame in default.frame]) + + def test_packet_data_stops_at_the_original_length(self) -> None: + from pcapkit.foundation.engines.dpkt import PCAPNGReader + + # the octet past the original length is padding, not packet data + capture = (_shb('<') + _idb('<', 1) + + _spb('<', ETHERNET + b'\x01', original_len=len(ETHERNET))) + with open(self._write(capture), 'rb') as file: + ((stamp, data),) = list(PCAPNGReader(file)) + self.assertEqual((data, stamp.orig_len), (ETHERNET, len(ETHERNET))) + + def test_a_raw_ipv4_interface_is_traced_as_by_the_default_engine(self) -> None: + # #1520 and #1548 at once: on a LINKTYPE_IPV4 interface, an SPB is read + # as a bare IP packet, which the flow tracer has to find too + import pcapkit + + self._write(_shb('<') + _idb('<', 228) + _spb('<', IPV4_TCP) + _epb('<', 0, IPV4_TCP)) + flows = {} + for engine in ('default', 'dpkt'): + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + extractor = pcapkit.extract(fin=self.path, engine=engine, nofile=True, tcp=True, + trace=True, trace_format='pcap', + trace_fout=os.path.join(self._tmp.name, engine)) + self.addCleanup(close_extractor, extractor) + flows[engine] = [] + for flow in extractor.trace.tcp: + with open(flow.fpout, 'rb') as file: + flows[engine].append((flow.label, tuple(flow.index), file.read())) + + self.assertEqual([index for _, index, _ in flows['dpkt']], [(1, 2)]) + self.assertEqual(flows['dpkt'], flows['default']) + + def test_a_section_without_an_interface_is_a_format_error(self) -> None: + from pcapkit.foundation.engines.dpkt import PCAPNGReader + from pcapkit.utilities.exceptions import FormatError + + with open(self._write(_shb('<') + _spb('<', ETHERNET)), 'rb') as file: + with self.assertRaisesRegex(FormatError, r'\[SPB\] section has no interface'): + next(PCAPNGReader(file)) + + if __name__ == '__main__': unittest.main() diff --git a/tests/foundation/engines/test_engine_agreement_runtime.py b/tests/foundation/engines/test_engine_agreement_runtime.py index 0689151950..402748af47 100644 --- a/tests/foundation/engines/test_engine_agreement_runtime.py +++ b/tests/foundation/engines/test_engine_agreement_runtime.py @@ -374,19 +374,6 @@ class TestEngineAgreement(harness.RoundTripBase): tuple(f'{capture}/pypcapfile/{aspect}' for capture in ('big_endian.pcap', 'big_endian_nanosecond.pcap') for aspect in ('frames', 'timestamps', 'ipv4', 'datagrams'))), - Gap(1501, 'the dpkt engine hands reassembly the Decimal timestamp dpkt reads from a ' - 'nanosecond PCAP (pcapkit/foundation/engines/dpkt.py:309), where every other ' - 'engine hands it a float (pcapkit/toolkit/pcap.py:73)', - 'INPUTS', "timestamp Decimal('1500000000.123456789') != 1500000000.1234567", - ('big_endian_nanosecond.pcap/dpkt/ipv4',)), - Gap(1520, "pcapkit's own PCAPNGReader for the dpkt engine yields only Enhanced and " - 'obsolete Packet Blocks, so a Simple Packet Block is dropped and every later ' - 'frame shifts by one (pcapkit/foundation/engines/dpkt.py:124)', - ('FRAMES', 'RECORDS', 'INPUTS', 'DATAGRAMS'), - ('4 frames != 5', 'frame 4: ', '2 datagrams != 3'), - ('test.pcapng/dpkt/frames', 'test.pcapng/dpkt/timestamps', 'test.pcapng/dpkt/octets', - 'test.pcapng/dpkt/ipv4', - 'test.pcapng/dpkt/datagrams')), Gap(1502, 'the default engine does not dissect LINKTYPE_RAW (101) frames, which scapy reads ' 'as IPv4: neither Frame.__proto__ nor PCAPNG.__proto__ registers LinkType.RAW ' '(pcapkit/protocols/misc/pcap/frame.py:90, pcapkit/protocols/misc/pcapng.py:699), ' diff --git a/tests/toolkit/test_dpkt_unit.py b/tests/toolkit/test_dpkt_unit.py index 7ee8de231e..81bccb3356 100644 --- a/tests/toolkit/test_dpkt_unit.py +++ b/tests/toolkit/test_dpkt_unit.py @@ -216,6 +216,84 @@ def test_tcp_reassembly_and_traceflow_accept_dpkt_data_payload_tcp(self) -> None self.assertIsNone(toolkit.tcp_traceflow(types.SimpleNamespace(ip=raw_ip), 1.0, data_link=LinkType.ETHERNET, count=1)) + def test_every_adapter_hands_on_a_float_timestamp(self) -> None: + # dpkt reads a nanosecond PCAP's timestamp as a Decimal; reassembly and + # flow tracing get a float from every engine, so from this one too (#1501) + import decimal + + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import dpkt as toolkit + + exact = decimal.Decimal('1500000000.123456789') + stamp = toolkit.DecimalTimestamp(exact, 1_000_000_000, 70) + packet = self._make_ipv4_tcp_packet() + toolkit.attach_timestamp(packet, stamp) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + results = { + 'ipv4': toolkit.ipv4_reassembly(self._make_ipv4_tcp_packet(fragmented=True), + stamp, count=1), + 'ipv6': toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=FakeIPv6(FakeFragment())), + stamp, count=1), + 'tcp': toolkit.tcp_reassembly(packet, stamp, count=1), + 'trace': toolkit.tcp_traceflow(packet, stamp, data_link=LinkType.ETHERNET, count=1), + } + for adapter, data in results.items(): + with self.subTest(adapter=adapter): + assert data is not None + self.assertIs(type(data.timestamp), float) + self.assertEqual(data.timestamp, float(exact)) + + # the PCAP frame record still carries every nanosecond + record = results['trace'].frame # type: ignore[union-attr] + self.assertEqual(record.time_epoch, exact) + self.assertEqual((record.frame_info.ts_sec, record.frame_info.ts_usec, + record.frame_info.orig_len), (1500000000, 123456789, 70)) + + def test_a_bare_ip_packet_is_its_own_network_layer(self) -> None: + # On a raw-IP link type dpkt reads a frame as a bare IP or IP6, which + # names no ``ip`` or ``ip6`` layer, so every adapter returned None (#1548) + import dpkt + + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import dpkt as toolkit + + def bare(layer): # type: ignore[no-untyped-def] + octets = bytes(layer) + packet = type(layer)(octets) + toolkit.attach_buffer(packet, octets) + return packet + + def fields(data): # type: ignore[no-untyped-def] + return {key: value for key, value in data.items() if key != 'frame'} + + framed = self._make_ipv4_tcp_packet() + toolkit.attach_buffer(framed, bytes(framed)) + ipv4 = bare(framed.ip) + self.assertEqual(fields(toolkit.tcp_reassembly(ipv4, 1.0, count=1)), + fields(toolkit.tcp_reassembly(framed, 1.0, count=1))) + self.assertEqual( + fields(toolkit.tcp_traceflow(ipv4, 1.0, data_link=LinkType.IPV4, count=1)), + fields(toolkit.tcp_traceflow(framed, 1.0, data_link=LinkType.IPV4, count=1))) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + fragment = self._make_ipv4_tcp_packet(fragmented=True) + self.assertEqual(toolkit.ipv4_reassembly(bare(fragment.ip), 1.0, count=1), + toolkit.ipv4_reassembly(fragment, 1.0, count=1)) + self.assertIsNone(toolkit.ipv6_reassembly(ipv4, 1.0, count=1)) + + ipv6 = bare(dpkt.ip6.IP6( + src=ip_address('2001:db8::1').packed, dst=ip_address('2001:db8::2').packed, + nxt=44, hlim=64, plen=8 + 20, + # Fragment header: offset 0, More Fragments set, Identification 99 + data=struct.pack('>BBHI', IP_PROTO_EXPERIMENTAL, 0, 1, 99) + bytes(20), + )) + reassembled = toolkit.ipv6_reassembly(ipv6, 1.0, count=1) + assert reassembled is not None + self.assertEqual((reassembled.bufid[2], reassembled.mf, bytes(reassembled.payload)), + (99, True, bytes(20))) + self.assertIsNone(toolkit.ipv4_reassembly(ipv6, 1.0, count=1)) + def test_tcp_helpers_cover_ipv6_and_data_payload_fallback(self) -> None: from pcapkit.const.reg.linktype import LinkType from pcapkit.toolkit import dpkt as toolkit diff --git a/tests/toolkit/test_engine_pcap_trace_runtime.py b/tests/toolkit/test_engine_pcap_trace_runtime.py index b1d9e7717d..78efefa957 100644 --- a/tests/toolkit/test_engine_pcap_trace_runtime.py +++ b/tests/toolkit/test_engine_pcap_trace_runtime.py @@ -147,16 +147,14 @@ def _flows(self, engine: 'str', fin: 'str', *, nanosecond: 'bool' = False, flows.append((flow.label, tuple(flow.index), file.read())) return flows - def _assert_same_files(self, fin: 'str', *, nanosecond: 'bool', labels: 'bool' = True, + def _assert_same_files(self, fin: 'str', *, nanosecond: 'bool', caught: 'list | None' = None) -> None: base = self._flows('default', fin, nanosecond=nanosecond) self.assertTrue(base) for engine in ENGINES: with self.subTest(engine=engine): flows = self._flows(engine, fin, nanosecond=nanosecond, caught=caught) - if labels: - self.assertEqual([flow[:2] for flow in flows], [flow[:2] for flow in base]) - self.assertEqual([flow[1] for flow in flows], [flow[1] for flow in base]) + self.assertEqual([flow[:2] for flow in flows], [flow[:2] for flow in base]) for (label, _, mine), (_, _, theirs) in zip(flows, base): self.assertEqual(mine[:4], MAGIC[nanosecond], label) self.assertEqual(mine, theirs, label) @@ -175,15 +173,31 @@ def test_traced_pcap_matches_the_default_engine_byte_for_byte(self) -> None: def test_a_nanosecond_pcap_keeps_its_resolution(self) -> None: # The record fraction stays in the capture's own resolution, as the default - # engine's does. Labels are left out: dpkt's Decimal timestamp spells them - # with nine places (#1501). + # engine's does. The labels agree too: the flow tracer is handed a float, + # not the Decimal dpkt reads a nanosecond PCAP's timestamp as (#1501). header, _ = _records(sample_path('tcp.pcap')) for case, extra in EXTRA_NS.items(): fin = _write(os.path.join(self.tmp, f'{case}.pcap'), MAGIC[True] + header[4:], self._nanosecond_records(extra)) for nanosecond in (False, True): with self.subTest(case=case, nanosecond=nanosecond): - self._assert_same_files(fin, nanosecond=nanosecond, labels=False) + self._assert_same_files(fin, nanosecond=nanosecond) + + @unittest.skipUnless(HAS_DPKT, 'dpkt not installed') + def test_a_raw_ip_link_type_traces_as_the_default_engine(self) -> None: + # With the Ethernet header stripped, dpkt reads each frame as a bare IP or + # IP6, which names no ``ip`` or ``ip6`` layer, so it traced nothing (#1548). + # Every frame is kept, IPv6 ones under LINKTYPE_IPV4 included. + for capture, linktype in (('tcp.pcap', 228), ('http6.cap', 229)): + header, records = _records(sample_path(capture)) + fin = _write(os.path.join(self.tmp, f'raw-{linktype}.pcap'), + header[:20] + struct.pack(' None: # ``if_tsresol=9``: an unaligned timestamp must keep its sub-microsecond