diff --git a/docs/source/pcapkit/foundation/engines/3rdparty.rst b/docs/source/pcapkit/foundation/engines/3rdparty.rst index 2efcc71e6a..10a712e211 100644 --- a/docs/source/pcapkit/foundation/engines/3rdparty.rst +++ b/docs/source/pcapkit/foundation/engines/3rdparty.rst @@ -64,26 +64,6 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. .. _DPKT: https://dpkt.readthedocs.io -.. important:: - - On PCAP-NG, `DPKT`_ yields **Enhanced Packet Blocks and (obsolete) Packet - Blocks only**. ``dpkt.pcapng.Reader.__iter__`` skips every other block type - without a warning (``# just ignore other blocks``, as of ``dpkt`` 1.9.8), and - :class:`~pcapkit.foundation.engines.dpkt.PCAPNGReader`, which this engine - reads PCAP-NG with, keeps that block set. It includes the **Simple Packet - Block**, which carries a packet just as an Enhanced Packet Block does. Such packets are silently missing from this - engine's output, and the frames after them are numbered as if they had never - been captured. - - Reading :file:`examples/captures/test.pcapng` -- four Enhanced Packet Blocks - and one Simple Packet Block, across two sections -- this engine gives four - frames where the default engine and - :class:`~pcapkit.foundation.engines.scapy.Scapy` give five; the missing one is - frame 4, the Simple Packet Block. The second Section Header Block is not the - cause: the Enhanced Packet Block after it is read. This is a limitation of - `DPKT`_, not of :mod:`pcapkit`, so read a capture that may contain Simple - Packet Blocks with the default engine instead. - .. note:: ``dpkt.pcapng.Reader`` reads only the **first** Interface Description Block @@ -93,6 +73,12 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. resolves each packet's link type, ``if_tsresol`` and ``if_tsoffset`` from the interface it names, in its own section (:issue:`1379`). + ``dpkt.pcapng.Reader`` also skips Simple Packet Blocks, which `DPKT`_ has no + type for; :class:`~pcapkit.foundation.engines.dpkt.PCAPNGReader` reads them + through :mod:`pcapkit`'s own :class:`~pcapkit.protocols.misc.pcapng.PCAPNG` + and, since the block carries no timestamp, dates them 0, as the default + engine does (:issue:`1520`). + .. autoclass:: pcapkit.foundation.engines.dpkt.DPKT :no-members: :show-inheritance: diff --git a/pcapkit/foundation/engines/dpkt.py b/pcapkit/foundation/engines/dpkt.py index 6139fe90d7..4a05b42a79 100644 --- a/pcapkit/foundation/engines/dpkt.py +++ b/pcapkit/foundation/engines/dpkt.py @@ -32,9 +32,11 @@ from pcapkit.foundation.engines.pcapng import Context from pcapkit.foundation.extraction import Extractor + from pcapkit.protocols.data.misc.pcapng import IF_TSResolOption as Data_IF_TSResolOption from pcapkit.protocols.data.misc.pcapng import \ InterfaceDescriptionBlock as Data_InterfaceDescriptionBlock from pcapkit.protocols.data.misc.pcapng import SectionHeaderBlock as Data_SectionHeaderBlock + from pcapkit.protocols.data.misc.pcapng import SimplePacketBlock as Data_SimplePacketBlock Reader = Union[PCAPReader, 'PCAPNGReader'] @@ -55,8 +57,11 @@ class PCAPNGReader: ``if_tsresol`` and ``if_tsoffset`` from the interface it names. Packet blocks are still decoded by `DPKT`_. - As :class:`dpkt.pcapng.Reader` does, it yields Enhanced Packet Blocks and - (obsolete) Packet Blocks only, and skips every other block. + It yields Enhanced Packet Blocks, (obsolete) Packet Blocks and Simple Packet + Blocks, and skips every other block. :class:`dpkt.pcapng.Reader` skips the + Simple Packet Block too, and `DPKT`_ has no type for one, so that block is + parsed with :class:`~pcapkit.protocols.misc.pcapng.PCAPNG` as well + (:issue:`1520`). .. _DPKT: https://dpkt.readthedocs.io @@ -123,6 +128,8 @@ def _read(self) -> 'Iterator[tuple[float, bytes]]': self._ctx.interfaces.append(cast('Data_InterfaceDescriptionBlock', idb.info)) elif block_type in (Enum_BlockType.Enhanced_Packet_Block, Enum_BlockType.Packet_Block): yield self._read_packet(buf, block_type, byteorder == 'little') + elif block_type == Enum_BlockType.Simple_Packet_Block: + yield self._read_simple(buf) def _read_packet(self, buf: 'bytes', block_type: 'int', little: 'bool') -> 'tuple[float, bytes]': @@ -177,6 +184,46 @@ def _read_packet(self, buf: 'bytes', block_type: 'int', return (Timestamp(float(timestamp), timestamp, resolution, pkt.pkt_len), # pylint: disable=no-member pkt.pkt_data) + def _read_simple(self, buf: 'bytes') -> 'tuple[float, bytes]': + """Decode a Simple Packet Block, against the section's first interface. + + `DPKT`_ has no type for this block, so it is parsed by + :class:`~pcapkit.protocols.misc.pcapng.PCAPNG`, as for the default + engine: the packet data is ``min(original_len, snaplen)`` octets, with + the snaplen of interface 0. + + Args: + buf: Whole block, as read from the file. + + Returns: + Timestamp and packet data, as :meth:`_read_packet` returns them. A + Simple Packet Block carries no timestamp, so the packet is dated at + the UNIX epoch, as the default engine dates it; the original length + is the block's own. + + Raises: + FormatError: If the section describes no interface. A Simple Packet + Block names none, so it refers to the first (Section 4.4 of the + PCAP-NG specification). + + """ + from pcapkit.protocols.misc.pcapng import PCAPNG as P_PCAPNG + from pcapkit.toolkit.dpkt import Timestamp # isort:skip + + if not self._ctx.interfaces: + raise FormatError('PCAP-NG: [SPB] section has no interface description block') + interface = self._ctx.interfaces[0] + block = P_PCAPNG(buf, num=0, sct=self._sect, ctx=self._ctx) + spb = cast('Data_SimplePacketBlock', block.info) + + tsresol = cast('Optional[Data_IF_TSResolOption]', + interface.options.get(Enum_OptionType.if_tsresol)) + resolution = 1_000_000 if tsresol is None else tsresol.resolution + + self._linktype = interface.linktype + return (Timestamp(0.0, decimal.Decimal(0), resolution, spb.original_len), + bytes(spb.packet)) + def _pcap_reader() -> 'Type[PCAPReader]': """A :class:`dpkt.pcap.Reader` that keeps each record's original length. @@ -258,10 +305,9 @@ class DPKT(EngineBase['DPKTPacket']): :class:`dpkt.pcapng.Reader`, so each packet takes the link type and the timestamp resolution and offset of the interface it was captured on. - Like `DPKT`_'s reader, it yields Enhanced Packet Blocks and (obsolete) - Packet Blocks only, and skips every other block without a warning -- the - Simple Packet Block included. A packet carried in a Simple Packet Block is - therefore missing from this engine's output; the default engine reads it. + It reads every packet block the default engine reads -- the Enhanced, the + Simple and the (obsolete) Packet Block -- and skips every other block + without a warning. `DPKT`_'s own reader skips the Simple Packet Block too. .. _DPKT: https://dpkt.readthedocs.io diff --git a/pcapkit/toolkit/dpkt.py b/pcapkit/toolkit/dpkt.py index 756afd3b46..b4d2d9a29d 100644 --- a/pcapkit/toolkit/dpkt.py +++ b/pcapkit/toolkit/dpkt.py @@ -81,8 +81,9 @@ class RecordTimestamp: resolution and its original length would be lost between the reader and :func:`packet2frame`. The engine's readers yield a :class:`Timestamp` or a :class:`DecimalTimestamp` instead, which *is* the value `DPKT`_ yields -- - so reassembly and flow labels see exactly what they did before -- and also - carries those three. + so a stored frame's timestamp is exactly what it was before -- and also + carries those three. Reassembly and flow tracing are handed a plain + :obj:`float` of it, as by every other engine (:issue:`1501`). .. _DPKT: https://dpkt.readthedocs.io @@ -395,6 +396,30 @@ def _layer2bytes(packet: 'Packet', layer: 'Packet') -> 'tuple[bytes, bool]': return packet2bytes(layer), False +def _network(packet: 'Packet') -> 'Optional[IP | IP6]': + """Fetch the network layer of a DPKT packet. + + Args: + packet: DPKT packet, i.e. the outermost layer. + + Returns: + The packet itself if it is an IPv4 or IPv6 packet, as `DPKT`_ reads every + frame of a raw-IP link type (:issue:`1548`); otherwise the IPv4 or IPv6 + payload of its link layer, which `DPKT`_ names ``ip`` or ``ip6`` -- a + bare IP packet names neither -- or :data:`None` if it has none. + + .. _DPKT: https://dpkt.readthedocs.io + + """ + import dpkt # isort:skip + + if isinstance(packet, (dpkt.ip.IP, dpkt.ip6.IP6)): + return packet + if hasattr(packet, 'ip'): + return cast('IP', packet.ip) + return getattr(packet, 'ip6', None) + + def ipv6_hdr_len(ipv6: 'IP6') -> 'int': """Calculate length of headers before IPv6 Fragment header. @@ -482,7 +507,9 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, than on the packet, so it is passed in -- as :func:`tcp_traceflow` does. A caller holding only a frame can read it back with :func:`packet2timestamp`, which is where - :class:`~pcapkit.foundation.engines.dpkt.DPKT` leaves it. + :class:`~pcapkit.foundation.engines.dpkt.DPKT` leaves it. Handed on + as a :obj:`float`, as every engine's adapters do, though `DPKT`_ reads + a nanosecond PCAP's as a :class:`~decimal.Decimal` (:issue:`1501`). count: Packet index. If not provided, default to ``-1``. Returns: @@ -507,7 +534,9 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.reassembly.ipv4.IPv4` """ - ipv4 = getattr(packet, 'ip', None) # type: Optional[IP] + network = _network(packet) + # an IPv6 layer is the one with extension headers, as in ``_wire_hdr_len`` + ipv4 = None if hasattr(network, 'extension_hdrs') else network if ipv4 is not None: if ipv4.df: # dismiss not fragmented packet return None @@ -537,7 +566,7 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, tl=ipv4.len, # total length, header includes header=wire[:ihl], # raw bytes type header payload=bytearray(wire[ihl:]), # raw bytearray type payload - timestamp=timestamp, # capture timestamp + timestamp=float(timestamp), # capture timestamp ) return data return None @@ -551,6 +580,7 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, packet: DPKT packet. timestamp: Capture timestamp of the packet, which drives the reassembly timeout; :func:`packet2timestamp` reads it back off a stored frame. + Handed on as a :obj:`float`, as for :func:`ipv4_reassembly`. count: Packet index. If not provided, default to ``-1``. Returns: @@ -575,7 +605,8 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.reassembly.ipv6.IPv6` """ - ipv6 = getattr(packet, 'ip6', None) # type: Optional[IP6] + network = _network(packet) + ipv6 = cast('IP6', network) if hasattr(network, 'extension_hdrs') else None if ipv6 is not None: ipv6_frag = ipv6.extension_hdrs.get(44) # type: Optional[IP6FragmentHeader] if ipv6_frag is None: # dismiss not fragmented packet @@ -618,7 +649,7 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, tl=hdr_len + len(payload), # total length, header includes header=wire[:hdr_len], # raw bytes type header before IPv6-Frag payload=bytearray(payload), # raw bytearray type payload after IPv6-Frag - timestamp=timestamp, # capture timestamp + timestamp=float(timestamp), # capture timestamp ) return data return None @@ -632,6 +663,7 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, packet: DPKT packet. timestamp: Capture timestamp of the packet, which drives the reassembly timeout; :func:`packet2timestamp` reads it back off a stored frame. + Handed on as a :obj:`float`, as for :func:`ipv4_reassembly`. count: Packet index. If not provided, default to ``-1``. Returns: @@ -655,11 +687,8 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.reassembly.tcp.TCP` """ - if hasattr(packet, 'ip'): - ip = cast('IP', packet.ip) - elif hasattr(packet, 'ip6'): - ip = cast('IP6', packet.ip6) - else: + ip = _network(packet) + if ip is None: return None tcp = getattr(ip, 'tcp', None) # type: Optional[TCP] @@ -688,7 +717,7 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, first=tcp.seq, # first sequence number of payload last=tcp.seq + raw_len - 1, # last sequence number of payload len=raw_len, # payload length, header excludes - timestamp=timestamp, # capture timestamp + timestamp=float(timestamp), # capture timestamp ) return data return None @@ -700,7 +729,10 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, Args: packet: DPKT packet. - timestamp: Timestamp of the packet. + timestamp: Timestamp of the packet, handed on as a :obj:`float`, as for + :func:`ipv4_reassembly`. The PCAP frame record is built from the + exact timestamp attached by :func:`attach_timestamp`, if any; see + :func:`packet2frame`. data_link: Data link layer protocol (from global header). count: Packet index. If not provided, default to ``-1``. @@ -725,11 +757,8 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.traceflow.tcp.TCP` """ - if hasattr(packet, 'ip'): - ip = cast('IP', packet.ip) - elif hasattr(packet, 'ip6'): - ip = cast('IP6', packet.ip6) - else: + ip = _network(packet) + if ip is None: return None tcp = getattr(ip, 'tcp', None) # type: Optional[TCP] @@ -750,7 +779,7 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, dst=ipaddress.ip_address(ip.dst), # destination IP srcport=tcp.sport, # TCP source port dstport=tcp.dport, # TCP destination port - timestamp=timestamp, # timestamp + timestamp=float(timestamp), # timestamp seq=tcp.seq, # TCP sequence number ack=tcp.ack, # TCP acknowledgement number header=wire[:tcp.off * 4], # raw bytes type header diff --git a/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py b/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py index 1fabd22040..bc9f1e7b4d 100644 --- a/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py +++ b/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py @@ -12,6 +12,9 @@ ``if_tsoffset``. Everything from :mod:`pcapkit` is imported inside each test, after :func:`~tests._support.reimport_once_per_class`, not at module load. +GitHub issue #1520: the engine's reader skipped Simple Packet Blocks, so their +packets were missing and every later frame was numbered one short. + """ import importlib.util @@ -21,12 +24,15 @@ import unittest import warnings -from tests._support import reimport_once_per_class +from tests._support import close_extractor, reimport_once_per_class HAS_DPKT = importlib.util.find_spec('dpkt') is not None #: Minimal IPv4 header (no payload), protocol 59 -- usable under any IP link type. IPV4 = bytes.fromhex('45000014000100004000') + bytes(2) + bytes([10, 0, 0, 1, 10, 0, 0, 2]) +#: IPv4 header (protocol 6) followed by a bare TCP SYN, from port 1234 to 80. +IPV4_TCP = (bytes.fromhex('45000028000100004006') + bytes(2) + bytes([10, 0, 0, 1, 10, 0, 0, 2]) + + struct.pack('>HHIIBBHHH', 1234, 80, 1, 0, 0x50, 0x02, 8192, 0, 0)) #: Ethernet frame carrying :data:`IPV4`. ETHERNET = bytes(6) + bytes([2, 0, 0, 0, 0, 1]) + b'\x08\x00' + IPV4 @@ -49,7 +55,7 @@ def _shb(order: str) -> bytes: def _idb(order: str, linktype: int, *, tsresol: 'int | None' = None, - tsoffset: 'int | None' = None) -> bytes: + tsoffset: 'int | None' = None, snaplen: int = 65535) -> bytes: options = b'' if tsresol is not None: options += _pad(struct.pack(f'{order}HHB', 9, 1, tsresol)) @@ -57,7 +63,7 @@ def _idb(order: str, linktype: int, *, tsresol: 'int | None' = None, options += struct.pack(f'{order}HHq', 14, 8, tsoffset) if options: options += struct.pack(f'{order}HH', 0, 0) - return _block(order, 1, struct.pack(f'{order}HHI', linktype, 0, 65535) + options) + return _block(order, 1, struct.pack(f'{order}HHI', linktype, 0, snaplen) + options) def _epb(order: str, iface: int, data: bytes) -> bytes: @@ -65,6 +71,11 @@ def _epb(order: str, iface: int, data: bytes) -> bytes: len(data), len(data)) + data) +def _spb(order: str, data: bytes, original_len: 'int | None' = None) -> bytes: + original_len = len(data) if original_len is None else original_len + return _block(order, 3, struct.pack(f'{order}I', original_len) + data) + + #: Packets of :func:`_capture`: (link type, parsed-as, timestamp), in file order. EXPECTED = [ (1, 'Ethernet', TICKS / 10**6), # section 1, interface 0 @@ -153,5 +164,108 @@ def test_unknown_interface_id_is_a_format_error(self) -> None: next(PCAPNGReader(file)) + +@unittest.skipUnless(HAS_DPKT, 'dpkt is not installed') +class TestDPKTSimplePacketBlock(unittest.TestCase): + """The DPKT engine reads a Simple Packet Block, as the default engine does.""" + + #: Ethernet, then raw IPv4 on nanoseconds; an SPB names no interface, so it + #: is on the first, whose snaplen cuts it short. Big-endian, so the block's + #: own fields are read in the section's byte order. + CAPTURE = b''.join([ + _shb('>'), + _idb('>', 1, snaplen=len(ETHERNET)), + _idb('>', 101, tsresol=9), + _epb('>', 1, IPV4), + _spb('>', ETHERNET, original_len=len(ETHERNET) + 40), + _epb('>', 0, ETHERNET), + ]) + + def setUp(self) -> None: + reimport_once_per_class(self) + self._tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + self.addCleanup(self._tmp.cleanup) + self.path = os.path.join(self._tmp.name, 'simple.pcapng') + + def _write(self, data: bytes) -> str: + with open(self.path, 'wb') as file: + file.write(data) + return self.path + + def test_reader_yields_the_block_on_the_first_interface(self) -> None: + import decimal + + from pcapkit.foundation.engines.dpkt import PCAPNGReader + + with open(self._write(self.CAPTURE), 'rb') as file: + reader = PCAPNGReader(file) + seen = [(int(reader.datalink()), timestamp, data) for timestamp, data in reader] + + self.assertEqual([(lt, data) for lt, _, data in seen], + [(101, IPV4), (1, ETHERNET), (1, ETHERNET)]) + # no timestamp in the block: dated at the epoch, as the default engine + # dates it; the original length is the block's, not the captured one, + # and the data stops at the snaplen, short of the block's padding + stamp = seen[1][1] + self.assertEqual((float(stamp), stamp.exact, stamp.orig_len), + (0.0, decimal.Decimal(0), len(ETHERNET) + 40)) + + def test_dpkt_engine_matches_default_engine(self) -> None: + import pcapkit + from pcapkit.toolkit.dpkt import packet2bytes, packet2timestamp + + self._write(self.CAPTURE) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + default = pcapkit.extract(fin=self.path, engine='default', nofile=True, store=True) + dpkt = pcapkit.extract(fin=self.path, engine='dpkt', nofile=True, store=True) + + self.assertEqual(dpkt.length, default.length) + self.assertEqual([(float(packet2timestamp(frame)), packet2bytes(frame)) + for frame in dpkt.frame], + [(float(frame.info.timestamp_epoch), bytes(frame.info.packet)) + for frame in default.frame]) + + def test_packet_data_stops_at_the_original_length(self) -> None: + from pcapkit.foundation.engines.dpkt import PCAPNGReader + + # the octet past the original length is padding, not packet data + capture = (_shb('<') + _idb('<', 1) + + _spb('<', ETHERNET + b'\x01', original_len=len(ETHERNET))) + with open(self._write(capture), 'rb') as file: + ((stamp, data),) = list(PCAPNGReader(file)) + self.assertEqual((data, stamp.orig_len), (ETHERNET, len(ETHERNET))) + + def test_a_raw_ipv4_interface_is_traced_as_by_the_default_engine(self) -> None: + # #1520 and #1548 at once: on a LINKTYPE_IPV4 interface, an SPB is read + # as a bare IP packet, which the flow tracer has to find too + import pcapkit + + self._write(_shb('<') + _idb('<', 228) + _spb('<', IPV4_TCP) + _epb('<', 0, IPV4_TCP)) + flows = {} + for engine in ('default', 'dpkt'): + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + extractor = pcapkit.extract(fin=self.path, engine=engine, nofile=True, tcp=True, + trace=True, trace_format='pcap', + trace_fout=os.path.join(self._tmp.name, engine)) + self.addCleanup(close_extractor, extractor) + flows[engine] = [] + for flow in extractor.trace.tcp: + with open(flow.fpout, 'rb') as file: + flows[engine].append((flow.label, tuple(flow.index), file.read())) + + self.assertEqual([index for _, index, _ in flows['dpkt']], [(1, 2)]) + self.assertEqual(flows['dpkt'], flows['default']) + + def test_a_section_without_an_interface_is_a_format_error(self) -> None: + from pcapkit.foundation.engines.dpkt import PCAPNGReader + from pcapkit.utilities.exceptions import FormatError + + with open(self._write(_shb('<') + _spb('<', ETHERNET)), 'rb') as file: + with self.assertRaisesRegex(FormatError, r'\[SPB\] section has no interface'): + next(PCAPNGReader(file)) + + if __name__ == '__main__': unittest.main() diff --git a/tests/foundation/engines/test_engine_agreement_runtime.py b/tests/foundation/engines/test_engine_agreement_runtime.py index 0689151950..402748af47 100644 --- a/tests/foundation/engines/test_engine_agreement_runtime.py +++ b/tests/foundation/engines/test_engine_agreement_runtime.py @@ -374,19 +374,6 @@ class TestEngineAgreement(harness.RoundTripBase): tuple(f'{capture}/pypcapfile/{aspect}' for capture in ('big_endian.pcap', 'big_endian_nanosecond.pcap') for aspect in ('frames', 'timestamps', 'ipv4', 'datagrams'))), - Gap(1501, 'the dpkt engine hands reassembly the Decimal timestamp dpkt reads from a ' - 'nanosecond PCAP (pcapkit/foundation/engines/dpkt.py:309), where every other ' - 'engine hands it a float (pcapkit/toolkit/pcap.py:73)', - 'INPUTS', "timestamp Decimal('1500000000.123456789') != 1500000000.1234567", - ('big_endian_nanosecond.pcap/dpkt/ipv4',)), - Gap(1520, "pcapkit's own PCAPNGReader for the dpkt engine yields only Enhanced and " - 'obsolete Packet Blocks, so a Simple Packet Block is dropped and every later ' - 'frame shifts by one (pcapkit/foundation/engines/dpkt.py:124)', - ('FRAMES', 'RECORDS', 'INPUTS', 'DATAGRAMS'), - ('4 frames != 5', 'frame 4: ', '2 datagrams != 3'), - ('test.pcapng/dpkt/frames', 'test.pcapng/dpkt/timestamps', 'test.pcapng/dpkt/octets', - 'test.pcapng/dpkt/ipv4', - 'test.pcapng/dpkt/datagrams')), Gap(1502, 'the default engine does not dissect LINKTYPE_RAW (101) frames, which scapy reads ' 'as IPv4: neither Frame.__proto__ nor PCAPNG.__proto__ registers LinkType.RAW ' '(pcapkit/protocols/misc/pcap/frame.py:90, pcapkit/protocols/misc/pcapng.py:699), ' diff --git a/tests/toolkit/test_dpkt_unit.py b/tests/toolkit/test_dpkt_unit.py index 7ee8de231e..81bccb3356 100644 --- a/tests/toolkit/test_dpkt_unit.py +++ b/tests/toolkit/test_dpkt_unit.py @@ -216,6 +216,84 @@ def test_tcp_reassembly_and_traceflow_accept_dpkt_data_payload_tcp(self) -> None self.assertIsNone(toolkit.tcp_traceflow(types.SimpleNamespace(ip=raw_ip), 1.0, data_link=LinkType.ETHERNET, count=1)) + def test_every_adapter_hands_on_a_float_timestamp(self) -> None: + # dpkt reads a nanosecond PCAP's timestamp as a Decimal; reassembly and + # flow tracing get a float from every engine, so from this one too (#1501) + import decimal + + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import dpkt as toolkit + + exact = decimal.Decimal('1500000000.123456789') + stamp = toolkit.DecimalTimestamp(exact, 1_000_000_000, 70) + packet = self._make_ipv4_tcp_packet() + toolkit.attach_timestamp(packet, stamp) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + results = { + 'ipv4': toolkit.ipv4_reassembly(self._make_ipv4_tcp_packet(fragmented=True), + stamp, count=1), + 'ipv6': toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=FakeIPv6(FakeFragment())), + stamp, count=1), + 'tcp': toolkit.tcp_reassembly(packet, stamp, count=1), + 'trace': toolkit.tcp_traceflow(packet, stamp, data_link=LinkType.ETHERNET, count=1), + } + for adapter, data in results.items(): + with self.subTest(adapter=adapter): + assert data is not None + self.assertIs(type(data.timestamp), float) + self.assertEqual(data.timestamp, float(exact)) + + # the PCAP frame record still carries every nanosecond + record = results['trace'].frame # type: ignore[union-attr] + self.assertEqual(record.time_epoch, exact) + self.assertEqual((record.frame_info.ts_sec, record.frame_info.ts_usec, + record.frame_info.orig_len), (1500000000, 123456789, 70)) + + def test_a_bare_ip_packet_is_its_own_network_layer(self) -> None: + # On a raw-IP link type dpkt reads a frame as a bare IP or IP6, which + # names no ``ip`` or ``ip6`` layer, so every adapter returned None (#1548) + import dpkt + + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import dpkt as toolkit + + def bare(layer): # type: ignore[no-untyped-def] + octets = bytes(layer) + packet = type(layer)(octets) + toolkit.attach_buffer(packet, octets) + return packet + + def fields(data): # type: ignore[no-untyped-def] + return {key: value for key, value in data.items() if key != 'frame'} + + framed = self._make_ipv4_tcp_packet() + toolkit.attach_buffer(framed, bytes(framed)) + ipv4 = bare(framed.ip) + self.assertEqual(fields(toolkit.tcp_reassembly(ipv4, 1.0, count=1)), + fields(toolkit.tcp_reassembly(framed, 1.0, count=1))) + self.assertEqual( + fields(toolkit.tcp_traceflow(ipv4, 1.0, data_link=LinkType.IPV4, count=1)), + fields(toolkit.tcp_traceflow(framed, 1.0, data_link=LinkType.IPV4, count=1))) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + fragment = self._make_ipv4_tcp_packet(fragmented=True) + self.assertEqual(toolkit.ipv4_reassembly(bare(fragment.ip), 1.0, count=1), + toolkit.ipv4_reassembly(fragment, 1.0, count=1)) + self.assertIsNone(toolkit.ipv6_reassembly(ipv4, 1.0, count=1)) + + ipv6 = bare(dpkt.ip6.IP6( + src=ip_address('2001:db8::1').packed, dst=ip_address('2001:db8::2').packed, + nxt=44, hlim=64, plen=8 + 20, + # Fragment header: offset 0, More Fragments set, Identification 99 + data=struct.pack('>BBHI', IP_PROTO_EXPERIMENTAL, 0, 1, 99) + bytes(20), + )) + reassembled = toolkit.ipv6_reassembly(ipv6, 1.0, count=1) + assert reassembled is not None + self.assertEqual((reassembled.bufid[2], reassembled.mf, bytes(reassembled.payload)), + (99, True, bytes(20))) + self.assertIsNone(toolkit.ipv4_reassembly(ipv6, 1.0, count=1)) + def test_tcp_helpers_cover_ipv6_and_data_payload_fallback(self) -> None: from pcapkit.const.reg.linktype import LinkType from pcapkit.toolkit import dpkt as toolkit diff --git a/tests/toolkit/test_engine_pcap_trace_runtime.py b/tests/toolkit/test_engine_pcap_trace_runtime.py index b1d9e7717d..78efefa957 100644 --- a/tests/toolkit/test_engine_pcap_trace_runtime.py +++ b/tests/toolkit/test_engine_pcap_trace_runtime.py @@ -147,16 +147,14 @@ def _flows(self, engine: 'str', fin: 'str', *, nanosecond: 'bool' = False, flows.append((flow.label, tuple(flow.index), file.read())) return flows - def _assert_same_files(self, fin: 'str', *, nanosecond: 'bool', labels: 'bool' = True, + def _assert_same_files(self, fin: 'str', *, nanosecond: 'bool', caught: 'list | None' = None) -> None: base = self._flows('default', fin, nanosecond=nanosecond) self.assertTrue(base) for engine in ENGINES: with self.subTest(engine=engine): flows = self._flows(engine, fin, nanosecond=nanosecond, caught=caught) - if labels: - self.assertEqual([flow[:2] for flow in flows], [flow[:2] for flow in base]) - self.assertEqual([flow[1] for flow in flows], [flow[1] for flow in base]) + self.assertEqual([flow[:2] for flow in flows], [flow[:2] for flow in base]) for (label, _, mine), (_, _, theirs) in zip(flows, base): self.assertEqual(mine[:4], MAGIC[nanosecond], label) self.assertEqual(mine, theirs, label) @@ -175,15 +173,31 @@ def test_traced_pcap_matches_the_default_engine_byte_for_byte(self) -> None: def test_a_nanosecond_pcap_keeps_its_resolution(self) -> None: # The record fraction stays in the capture's own resolution, as the default - # engine's does. Labels are left out: dpkt's Decimal timestamp spells them - # with nine places (#1501). + # engine's does. The labels agree too: the flow tracer is handed a float, + # not the Decimal dpkt reads a nanosecond PCAP's timestamp as (#1501). header, _ = _records(sample_path('tcp.pcap')) for case, extra in EXTRA_NS.items(): fin = _write(os.path.join(self.tmp, f'{case}.pcap'), MAGIC[True] + header[4:], self._nanosecond_records(extra)) for nanosecond in (False, True): with self.subTest(case=case, nanosecond=nanosecond): - self._assert_same_files(fin, nanosecond=nanosecond, labels=False) + self._assert_same_files(fin, nanosecond=nanosecond) + + @unittest.skipUnless(HAS_DPKT, 'dpkt not installed') + def test_a_raw_ip_link_type_traces_as_the_default_engine(self) -> None: + # With the Ethernet header stripped, dpkt reads each frame as a bare IP or + # IP6, which names no ``ip`` or ``ip6`` layer, so it traced nothing (#1548). + # Every frame is kept, IPv6 ones under LINKTYPE_IPV4 included. + for capture, linktype in (('tcp.pcap', 228), ('http6.cap', 229)): + header, records = _records(sample_path(capture)) + fin = _write(os.path.join(self.tmp, f'raw-{linktype}.pcap'), + header[:20] + struct.pack(' None: # ``if_tsresol=9``: an unaligned timestamp must keep its sub-microsecond