From b0af61f5be86c9e45cefcf42c8cfeeb50e0d1ddf Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 7 Oct 2026 14:46:38 -0400 Subject: [PATCH] fix(toolkit): dpkt reassembly adapters build their data from dpkt's re-serialised bytes and mutate the packet The DPKT engine now attaches each record's octets to the frame beside its timestamp (attach_buffer / packet2bytes). ipv4_reassembly, ipv6_reassembly, tcp_reassembly, tcp_traceflow and packet2dict slice the header and payload out of those octets instead of calling .pack(), which recomputed zeroed checksums and lengths and wrote them back into the packet. A hand-built packet with no buffer attached is serialised from a deep copy, so it is never mutated either. The layer's offset in those octets comes from the advance dpkt's own unpack() made, not from len(): IP6.__len__ sums len() of the extension headers, which overshoots by the rest of the frame for an Authentication header and falls 8 octets short for a Routing header whose Hdr Ext Len is odd. The slice is then checked against the layer's own pack_hdr() and a ProtocolError raised rather than wrong bytes returned. ipv6_hdr_len walks the headers in parsed order for the same reason, so an Authentication header before the Fragment header is counted and one after it is not. Closes #1297 --- pcapkit/foundation/engines/dpkt.py | 9 +- pcapkit/toolkit/dpkt.py | 244 ++++++++++- .../test_dpkt_reassembly_rawbytes_unit.py | 393 ++++++++++++++++++ 3 files changed, 626 insertions(+), 20 deletions(-) create mode 100644 tests/toolkit/test_dpkt_reassembly_rawbytes_unit.py diff --git a/pcapkit/foundation/engines/dpkt.py b/pcapkit/foundation/engines/dpkt.py index 9ea63ebf5b..bbbd12076e 100644 --- a/pcapkit/foundation/engines/dpkt.py +++ b/pcapkit/foundation/engines/dpkt.py @@ -150,8 +150,9 @@ def read_frame(self) -> 'DPKTPacket': for more operational information. """ - from pcapkit.toolkit.dpkt import (attach_timestamp, ipv4_reassembly, ipv6_reassembly, - packet2dict, tcp_reassembly, tcp_traceflow) + from pcapkit.toolkit.dpkt import (attach_buffer, attach_timestamp, ipv4_reassembly, + ipv6_reassembly, packet2dict, tcp_reassembly, + tcp_traceflow) ext = self._extractor reader = self._extmp @@ -167,6 +168,10 @@ def read_frame(self) -> 'DPKTPacket': # captured -- and anything reading a *stored* frame after this loop has # moved on could not find out. Keep the two together from the start. attach_timestamp(packet, timestamp) + # Nor does the packet keep the octets it was parsed from, and serialising + # it again recomputes zeroed checksums and lengths into it, so keep those + # too for the adapters to slice the wire octets out of. + attach_buffer(packet, pkt) # verbose output ext._frnum += 1 diff --git a/pcapkit/toolkit/dpkt.py b/pcapkit/toolkit/dpkt.py index 23f81c614a..cd781ff6db 100644 --- a/pcapkit/toolkit/dpkt.py +++ b/pcapkit/toolkit/dpkt.py @@ -12,6 +12,7 @@ .. _DPKT: https://dpkt.readthedocs.io """ +import copy import ipaddress from typing import TYPE_CHECKING, cast @@ -19,7 +20,7 @@ from pcapkit.foundation.reassembly.data.ip import Packet as IP_Packet from pcapkit.foundation.reassembly.data.tcp import Packet as TCP_Packet from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet -from pcapkit.utilities.exceptions import UnsupportedCall +from pcapkit.utilities.exceptions import ProtocolError, UnsupportedCall if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address @@ -33,7 +34,8 @@ from pcapkit.const.reg.linktype import LinkType as Enum_LinkType __all__ = [ - 'ipv6_hdr_len', 'attach_timestamp', 'packet2timestamp', 'packet2chain', 'packet2dict', + 'ipv6_hdr_len', 'attach_timestamp', 'packet2timestamp', 'attach_buffer', 'packet2bytes', + 'packet2chain', 'packet2dict', 'ipv4_reassembly', 'ipv6_reassembly', 'tcp_reassembly', 'tcp_traceflow' ] @@ -53,6 +55,19 @@ #: .. _DPKT: https://dpkt.readthedocs.io TIMESTAMP_ATTR = '__pcapkit_timestamp__' +#: Attribute a frame's record octets are stashed under. +#: +#: A `DPKT`_ packet keeps no copy of the octets it was parsed from, and +#: serialising it again is not the same thing: :meth:`dpkt.ip.IP.__bytes__` +#: recomputes a zeroed checksum and length and writes them back into the +#: packet, as do the transport layers beneath it. So the adapters below slice +#: the wire octets out of this buffer instead, which +#: :class:`~pcapkit.foundation.engines.dpkt.DPKT` attaches alongside +#: :data:`TIMESTAMP_ATTR`. +#: +#: .. _DPKT: https://dpkt.readthedocs.io +BUFFER_ATTR = '__pcapkit_buffer__' + def attach_timestamp(packet: 'Packet', timestamp: 'float') -> 'None': """Stash a frame's capture timestamp on the frame. @@ -94,6 +109,147 @@ def packet2timestamp(packet: 'Packet') -> 'float': return cast('float', timestamp) +def attach_buffer(packet: 'Packet', buffer: 'bytes') -> 'None': + """Stash a frame's record octets on the frame. + + Args: + packet: DPKT packet. + buffer: Octets of the record the packet was parsed from, as `DPKT`_'s + reader yielded them. + + .. _DPKT: https://dpkt.readthedocs.io + + """ + setattr(packet, BUFFER_ATTR, bytes(buffer)) + + +def packet2bytes(packet: 'Packet') -> 'bytes': + """Fetch the octets of a DPKT packet without modifying it. + + Args: + packet: DPKT packet. + + Returns: + The record octets attached by :func:`attach_buffer`, i.e. the frame as + it was captured. A packet that carries none -- one built by hand rather + than read by :class:`~pcapkit.foundation.engines.dpkt.DPKT` -- is + serialised from a copy instead, so that `DPKT`_'s checksum and length + recomputation never reaches the packet itself. + + .. _DPKT: https://dpkt.readthedocs.io + + """ + buffer = getattr(packet, BUFFER_ATTR, None) + if buffer is not None: + return cast('bytes', buffer) + return cast('bytes', copy.deepcopy(packet).pack()) + + +def _ipv6_ext_hdrs(ipv6: 'IP6') -> 'list[Any]': + """Fetch an IPv6 packet's extension headers, in the order they were parsed. + + Args: + ipv6: DPKT IPv6 packet. + + Returns: + :attr:`dpkt.ip6.IP6.all_extension_headers` where `DPKT`_ recorded it, + which keeps the on-wire order and any header type that occurs twice; + otherwise the values of :attr:`~dpkt.ip6.IP6.extension_hdrs`, which + :meth:`dpkt.ip6.IP6.__len__` falls back to in the same way. + + .. _DPKT: https://dpkt.readthedocs.io + + """ + ext_hdrs = getattr(ipv6, 'all_extension_headers', None) + if ext_hdrs: + return list(ext_hdrs) + return list(getattr(ipv6, 'extension_hdrs', {}).values()) + + +def _wire_hdr_len(packet: 'Packet', payload: 'Packet') -> 'int': + """Calculate the octets a layer occupies before its payload begins. + + Args: + packet: DPKT packet, i.e. one layer of a frame. + payload: That layer's payload, i.e. :attr:`packet.data `. + + Returns: + The advance :meth:`~dpkt.dpkt.Packet.unpack` itself made to reach the + payload. + + For IPv6 that is the fixed header plus each extension header's + :attr:`~dpkt.ip6.IP6ExtensionHeader.length`, which is what + :meth:`dpkt.ip6.IP6.unpack` walks the buffer by. It cannot be taken from + ``len()``: :meth:`dpkt.ip6.IP6.__len__` sums ``len()`` of the extension + headers instead, and that is 12 octets plus the *rest of the frame* for + an Authentication header -- :class:`dpkt.ip6.IP6AHHeader` trims only into + :attr:`~dpkt.ip6.IP6AHHeader.auth_data` and leaves + :attr:`~dpkt.dpkt.Packet.data` running to the end of the buffer -- and 8 + octets short of the wire for a Routing header whose Hdr Ext Len is odd + (:class:`dpkt.ip6.IP6RoutingHeader` keeps whole addresses only). + + Every other layer's ``len()`` does bound its payload, so the difference + of the two is used there. For IPv4 that difference is the Internet + Header Length, options included. + + """ + if hasattr(packet, 'extension_hdrs'): + return packet.__hdr_len__ + sum( + ext_hdr.length for ext_hdr in _ipv6_ext_hdrs(cast('IP6', packet)) + ) + return len(packet) - len(payload) + + +def _layer2bytes(packet: 'Packet', layer: 'Packet') -> 'tuple[bytes, bool]': + """Fetch the octets of a layer of a DPKT packet, from its first octet on. + + Args: + packet: DPKT packet, i.e. the outermost layer. + layer: A layer nested within ``packet``. + + Returns: + The octets, and whether they are the captured ones. With a buffer + attached by :func:`attach_buffer` they are the record octets from the + layer's offset to the end of the frame, so any link-layer trailer is + still there for the caller to trim; otherwise they are the layer alone, + serialised from a copy (see :func:`packet2bytes`). + + Raises: + ProtocolError: If the buffer does not carry the layer's own header where + the walk above placed it. `DPKT`_ unpacks a header field by field and + :meth:`~dpkt.dpkt.Packet.pack_hdr` packs those fields back without + recomputing any of them, so the two agree octet for octet wherever + the offset is right -- which makes this an exact check rather than a + plausibility one. It is raised rather than worked around because a + misplaced slice reaches the reassembler as a believable datagram and + so is invisible in its output. + + .. _DPKT: https://dpkt.readthedocs.io + + """ + buffer = getattr(packet, BUFFER_ATTR, None) + if buffer is not None: + offset = 0 + current = packet + while current is not layer: + payload = getattr(current, 'data', None) + if payload is None or isinstance(payload, (bytes, bytearray)): + break + offset += _wire_hdr_len(current, payload) + current = payload + else: + wire = cast('bytes', buffer)[offset:] + hdr_len = getattr(layer, '__hdr_len__', 0) + if hdr_len and wire[:hdr_len] != layer.pack_hdr(): + raise ProtocolError( + f'{type(layer).__name__} header not found at offset {offset} of the ' + f'{len(cast("bytes", buffer))}-octet record; the frame cannot be ' + 'sliced without re-serialising it' + ) + return wire, True + return packet2bytes(layer), False + + def ipv6_hdr_len(ipv6: 'IP6') -> 'int': """Calculate length of headers before IPv6 Fragment header. @@ -104,18 +260,22 @@ def ipv6_hdr_len(ipv6: 'IP6') -> 'int': Length of headers before IPv6 Fragment header :class:`dpkt.ip6.IP6FragmentHeader` (:rfc:`2460#section-4.5`). - As specified in :rfc:`2460#section-4.1`, such headers (before the IPv6 Fragment Header) - includes Hop-by-Hop Options header :class:`dpkt.ip6.IP6HopOptsHeader` (:rfc:`2460#section-4.3`), - Destination Options header :class:`dpkt.ip6.IP6DstOptHeader` (:rfc:`2460#section-4.6`) and - Routing header :class:`dpkt.ip6.IP6RoutingHeader` (:rfc:`2460#section-4.4`). + As specified in :rfc:`2460#section-4.1`, such a header may be a Hop-by-Hop Options + header :class:`dpkt.ip6.IP6HopOptsHeader` (:rfc:`2460#section-4.3`), a Destination + Options header :class:`dpkt.ip6.IP6DstOptsHeader` (:rfc:`2460#section-4.6`) or a + Routing header :class:`dpkt.ip6.IP6RoutingHeader` (:rfc:`2460#section-4.4`). An + Authentication header (:rfc:`4302`) reaches here as well, though, so the headers are + walked in the order they were parsed rather than looked up by a fixed list of types. + Walking them is also what counts a header type occurring twice twice, and what leaves + out one that sits *after* the Fragment header rather than before it. """ + frag = ipv6.extension_hdrs.get(44) hdr_len = ipv6.__hdr_len__ - # IP6HopOptsHeader / IP6DstOptHeader / IP6RoutingHeader - for code in (0, 60, 43): - ext_hdr = ipv6.extension_hdrs.get(code) - if ext_hdr is not None: - hdr_len += ext_hdr.length + for ext_hdr in _ipv6_ext_hdrs(ipv6): + if ext_hdr is frag: + break + hdr_len += ext_hdr.length return hdr_len @@ -161,7 +321,7 @@ def wrapper(packet: 'Packet') -> 'dict[str, Any]': return { 'timestamp': timestamp, - 'packet': packet.pack(), + 'packet': packet2bytes(packet), data_link.name: wrapper(packet), } @@ -189,6 +349,15 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for IPv4 reassembly (:term:`reasm.ipv4.packet`) will be returned; otherwise, returns :data:`None`. + Note: + The header and payload are sliced out of the record octets + :class:`~pcapkit.foundation.engines.dpkt.DPKT` attached to the frame, so + they are the captured ones. A packet built by hand carries no such + buffer, and its octets are then :mod:`dpkt`'s re-serialisation of it, in + which a zeroed checksum or length comes back recomputed -- + :func:`attach_buffer` is what makes them the wire's again. The packet + itself is left untouched either way. + See Also: :class:`pcapkit.foundation.reassembly.ipv4.IPv4` @@ -201,6 +370,11 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, # covers any IP options, whereas ``IP.__hdr_len__`` is the fixed 20-octet # struct size and would leave option octets at the head of the payload ihl = ipv4.hl * 4 + wire, captured = _layer2bytes(packet, ipv4) + # a zero Total Length is what TCP segmentation offload leaves behind, in + # which case the datagram runs to the end of the frame, as with DPKT + if captured and ipv4.len: + wire = wire[:ipv4.len] data = IP_Packet( bufid=( @@ -216,8 +390,8 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, ihl=ihl, # internet header length mf=bool(ipv4.mf), # more fragment flag tl=ipv4.len, # total length, header includes - header=ipv4.pack()[:ihl], # raw bytes type header - payload=bytearray(ipv4.pack()[ihl:]), # raw bytearray type payload + header=wire[:ihl], # raw bytes type header + payload=bytearray(wire[ihl:]), # raw bytearray type payload timestamp=timestamp, # capture timestamp ) return data @@ -243,6 +417,15 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for IPv6 reassembly (:term:`reasm.ipv6.packet`) will be returned; otherwise, returns :data:`None`. + Note: + The header and payload are sliced out of the record octets + :class:`~pcapkit.foundation.engines.dpkt.DPKT` attached to the frame, so + they are the captured ones. A packet built by hand carries no such + buffer, and its octets are then :mod:`dpkt`'s re-serialisation of it, in + which a zeroed checksum or length comes back recomputed -- + :func:`attach_buffer` is what makes them the wire's again. The packet + itself is left untouched either way. + See Also: :class:`pcapkit.foundation.reassembly.ipv6.IPv6` @@ -253,8 +436,13 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, if ipv6_frag is None: # dismiss not fragmented packet return None hdr_len = ipv6_hdr_len(ipv6) + wire, captured = _layer2bytes(packet, ipv6) + # a zero Payload Length is a jumbogram or TCP segmentation offload, in + # which case the datagram runs to the end of the frame, as with DPKT + if captured and ipv6.plen: + wire = wire[:ipv6.__hdr_len__ + ipv6.plen] # payload following the IPv6 Fragment header - payload = ipv6.pack()[hdr_len + ipv6_frag.__hdr_len__:] + payload = wire[hdr_len + ipv6_frag.__hdr_len__:] data = IP_Packet( bufid=( @@ -283,7 +471,7 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, ihl=hdr_len, # header length, only headers before IPv6-Frag mf=bool(ipv6_frag.m_flag), # more fragment flag tl=hdr_len + len(payload), # total length, header includes - header=ipv6.pack()[:hdr_len], # raw bytes type header before IPv6-Frag + header=wire[:hdr_len], # raw bytes type header before IPv6-Frag payload=bytearray(payload), # raw bytearray type payload after IPv6-Frag timestamp=timestamp, # capture timestamp ) @@ -309,6 +497,15 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP reassembly (:term:`reasm.tcp.packet`) will be returned; otherwise, returns :data:`None`. + Note: + The header and payload are sliced out of the record octets + :class:`~pcapkit.foundation.engines.dpkt.DPKT` attached to the frame, so + they are the captured ones. A packet built by hand carries no such + buffer, and its octets are then :mod:`dpkt`'s re-serialisation of it, in + which a zeroed checksum or length comes back recomputed -- + :func:`attach_buffer` is what makes them the wire's again. The packet + itself is left untouched either way. + See Also: :class:`pcapkit.foundation.reassembly.tcp.TCP` @@ -325,6 +522,7 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, tcp = cast('TCP', ip.data) if tcp is not None: flags = bin(tcp.flags)[2:].zfill(8) + wire, _ = _layer2bytes(packet, tcp) raw_len = len(tcp.data) # payload length, header excludes data = TCP_Packet( @@ -340,7 +538,7 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, rst=bool(int(flags[5])), # reset connection flag syn=bool(int(flags[6])), # synchronise flag fin=bool(int(flags[7])), # finish flag - header=tcp.pack()[:tcp.off * 4], # raw bytes type header + header=wire[:tcp.off * 4], # raw bytes type header payload=bytearray(bytes(tcp.data)), # raw bytearray type payload first=tcp.seq, # first sequence number of payload last=tcp.seq + raw_len - 1, # last sequence number of payload @@ -369,6 +567,15 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`. + Note: + The header and payload are sliced out of the record octets + :class:`~pcapkit.foundation.engines.dpkt.DPKT` attached to the frame, so + they are the captured ones. A packet built by hand carries no such + buffer, and its octets are then :mod:`dpkt`'s re-serialisation of it, in + which a zeroed checksum or length comes back recomputed -- + :func:`attach_buffer` is what makes them the wire's again. The packet + itself is left untouched either way. + See Also: :class:`pcapkit.foundation.traceflow.tcp.TCP` @@ -385,6 +592,7 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, tcp = cast('TCP', ip.data) if tcp is not None: flags = bin(tcp.flags)[2:].zfill(8) + wire, _ = _layer2bytes(packet, tcp) data = TF_TCP_Packet( # type: ignore[type-var] protocol=data_link, # data link type from global header @@ -400,7 +608,7 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, timestamp=timestamp, # timestamp seq=tcp.seq, # TCP sequence number ack=tcp.ack, # TCP acknowledgement number - header=tcp.pack()[:tcp.off * 4], # raw bytes type header + header=wire[:tcp.off * 4], # raw bytes type header payload=bytearray(bytes(tcp.data)), # raw bytearray type payload ) return data diff --git a/tests/toolkit/test_dpkt_reassembly_rawbytes_unit.py b/tests/toolkit/test_dpkt_reassembly_rawbytes_unit.py new file mode 100644 index 0000000000..db18d9ddd8 --- /dev/null +++ b/tests/toolkit/test_dpkt_reassembly_rawbytes_unit.py @@ -0,0 +1,393 @@ +"""The DPKT adapters hand on the wire octets, and leave the packet alone. + +GitHub issue #1297. :mod:`pcapkit.toolkit.dpkt` built the header and payload of +every reassembly and flow-tracing record by calling ``.pack()`` on the DPKT +layer. DPKT recomputes a zeroed checksum or length while packing *and writes it +back into the packet*, so the records no longer matched the capture, and the +first adapter to run changed what every later one saw. The adapters now slice the +record octets that :class:`~pcapkit.foundation.engines.dpkt.DPKT` attaches to the +frame. + +Slicing needs the layer's offset in those octets, and that cannot be read off +``len()``: :meth:`dpkt.ip6.IP6.__len__` sums ``len()`` of the extension headers +rather than their wire lengths, which is 12 octets plus the rest of the frame for +an Authentication header and 8 octets short for a Routing header whose Hdr Ext +Len is odd. :class:`~dpkt.ip6.IP6ExtensionHeader` records the wire length in +``length``, which is what :meth:`dpkt.ip6.IP6.unpack` walks the buffer by, so +that is what the offset is built from -- and the slice is then checked against +the layer's own header before it is handed back. + +Every frame below is written to a PCAP file and read back through the engine, so +the tests exercise the path a capture actually takes rather than a packet with a +buffer bolted on by hand. :class:`DPKTUnbufferedFallbackTests` is the exception, +and says why. + +""" +from __future__ import annotations + +import importlib.util +import os +import struct +import tempfile +import unittest +import warnings + +from tests._support import close_extractor, reimport_once_per_class, sample_path + +HAS_DPKT = importlib.util.find_spec('dpkt') is not None +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) + +#: Ethernet header, destination then source then EtherType. +ETH_IPV4 = b'\xbb' * 6 + b'\xaa' * 6 + b'\x08\x00' +ETH_IPV6 = b'\xbb' * 6 + b'\xaa' * 6 + b'\x86\xdd' + +#: IPv4 carrying SCTP, with a zero header checksum and a Total Length (0x38) +#: that DPKT's own serialisation would rewrite. The issue's own repro. +IPV4_SCTP = (bytes.fromhex('450000380000000040840000c0a80001c0a80002') + + bytes.fromhex('c35000501122334400000000' '00030013') + + b'\x41' * 15 + b'\x00' * 5) +#: IPv4 carrying TCP with a payload, both checksums zero -- the pair DPKT +#: recomputes together, in :meth:`dpkt.ip.IP.__bytes__`. +IPV4_TCP = (bytes.fromhex('4500002c00010000400600000a0000010a000002') + + struct.pack('>HHIIBBHHH', 1234, 80, 10, 5, 5 << 4, 0x18, 1024, 0, 0) + + b'data') +#: What ``IPV4_TCP`` comes back as once DPKT has re-serialised it: the IPv4 +#: header checksum and the TCP checksum both filled in. Pinned so that the +#: unbuffered fallback is a measured value rather than an assumption. +IPV4_TCP_REPACKED = (bytes.fromhex('4500002c00010000400666c90a0000010a000002') + + struct.pack('>HHIIBBHHH', 1234, 80, 10, 5, 5 << 4, 0x18, 1024, 0xb9d2, 0) + + b'data') + +#: TCP segment used by every extension-header frame below, checksum zero. +TCP_SEGMENT = struct.pack('>HHIIBBHHH', 1000, 2000, 1, 2, 5 << 4, 0x18, 100, 0, 0) + b'X' * 30 +#: Fragment header (:rfc:`8200#section-4.5`) whose Next Header is TCP. +FRAG_HDR = struct.pack('>BBHI', 6, 0, 0x0001, 0x1234) + + +def ipv6(payload: 'bytes', nxt: 'int') -> 'bytes': + """Build an IPv6 header over ``payload``, whose Next Header is ``nxt``.""" + return struct.pack('>IHBB16s16s', 0x60000000, len(payload), nxt, 64, + b'\x20\x01' + b'\x00' * 13 + b'\x01', + b'\x20\x01' + b'\x00' * 13 + b'\x02') + payload + + +def opts_hdr(nxt: 'int', ext_len: 'int' = 0) -> 'bytes': + """Build a Hop-by-Hop or Destination Options header (:rfc:`8200#section-4.3`). + + ``ext_len`` is the Hdr Ext Len field, i.e. the length in 8-octet units not + counting the first 8, so the header is ``(ext_len + 1) * 8`` octets. The body + is one PadN option filling the rest. + + """ + body_len = (ext_len + 1) * 8 - 4 + return struct.pack('>BBBB', nxt, ext_len, 1, body_len) + b'\x00' * body_len + + +def routing_hdr(nxt: 'int', ext_len: 'int') -> 'bytes': + """Build a Routing header (:rfc:`8200#section-4.4`) of ``ext_len`` 8-octet units. + + An *odd* ``ext_len`` is the case that matters: DPKT keeps whole 16-octet + addresses, so ``len()`` of the parsed header comes out 8 octets short of the + ``ext_len * 8 + 8`` the wire declares. + + """ + return struct.pack('>BBBBI', nxt, ext_len, 4, 1, 0) + b'\x01' * (ext_len * 8) + + +def ah_hdr(nxt: 'int', ext_len: 'int' = 4) -> 'bytes': + """Build an Authentication header (:rfc:`4302#section-2`). + + ``ext_len`` is the Payload Length field: the header's length in 4-octet units + minus two, so the header is ``(ext_len + 2) * 4`` octets. DPKT leaves + :attr:`dpkt.dpkt.Packet.data` untrimmed here, so ``len()`` of the parsed + header swallows everything that follows it in the frame. + + """ + return (struct.pack('>BBHII', nxt, ext_len, 0, 1, 2) + + b'\xaa' * ((ext_len + 2) * 4 - 12)) + + +#: ``(label, frame, wire IPv4/IPv6 datagram)`` for the plain cases. +FRAMES = ( + ('ipv4 sctp', ETH_IPV4 + IPV4_SCTP, IPV4_SCTP), + ('ipv4 tcp', ETH_IPV4 + IPV4_TCP, IPV4_TCP), + ('ipv6 frag udp', ETH_IPV6 + ipv6( + struct.pack('>BBHI', 17, 0, 0x0001, 4321) + + struct.pack('>HHHH', 53, 53, 12, 0) + b'udp!', 44), None), + ('ipv6 tcp', ETH_IPV6 + ipv6(TCP_SEGMENT, 6), None), +) + +#: ``(label, frame)`` for one TCP segment behind each extension header DPKT can +#: parse, including both the headers whose ``len()`` disagrees with the wire and +#: the ones whose does not, so a fix that broke the agreeing cases is visible. +EXT_HDR_FRAMES = ( + ('hop-by-hop', ETH_IPV6 + ipv6(opts_hdr(6) + TCP_SEGMENT, 0)), + ('hop-by-hop len 1', ETH_IPV6 + ipv6(opts_hdr(6, 1) + TCP_SEGMENT, 0)), + ('destination options', ETH_IPV6 + ipv6(opts_hdr(6) + TCP_SEGMENT, 60)), + ('destination options twice', + ETH_IPV6 + ipv6(opts_hdr(60) + opts_hdr(6) + TCP_SEGMENT, 60)), + ('routing len 2', ETH_IPV6 + ipv6(routing_hdr(6, 2) + TCP_SEGMENT, 43)), + ('routing len 3 (odd)', ETH_IPV6 + ipv6(routing_hdr(6, 3) + TCP_SEGMENT, 43)), + ('routing len 4', ETH_IPV6 + ipv6(routing_hdr(6, 4) + TCP_SEGMENT, 43)), + ('authentication', ETH_IPV6 + ipv6(ah_hdr(6) + TCP_SEGMENT, 51)), + ('authentication len 5', ETH_IPV6 + ipv6(ah_hdr(6, 5) + TCP_SEGMENT, 51)), + ('authentication len 7', ETH_IPV6 + ipv6(ah_hdr(6, 7) + TCP_SEGMENT, 51)), + ('authentication then destination options', + ETH_IPV6 + ipv6(ah_hdr(60) + opts_hdr(6) + TCP_SEGMENT, 51)), + ('hop-by-hop then routing len 3', + ETH_IPV6 + ipv6(opts_hdr(43) + routing_hdr(6, 3) + TCP_SEGMENT, 0)), +) + +#: An IPv6 fragment sitting behind an Authentication header, and the length of +#: the headers that precede its Fragment header: 40 + 24. +AH_FRAG_FRAME = ETH_IPV6 + ipv6(ah_hdr(44) + FRAG_HDR + TCP_SEGMENT, 51) +AH_FRAG_IHL = 40 + 24 + + +def write_pcap(frames: 'tuple[bytes, ...]') -> 'str': + """Write ``frames`` to a temporary Ethernet PCAP file and return its path.""" + fd, path = tempfile.mkstemp(suffix='.pcap') + with os.fdopen(fd, 'wb') as file: + file.write(struct.pack(' None: + reimport_once_per_class(self) + + def _extract(self, path: 'str', engine: 'str' = 'dpkt'): + import pcapkit + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + extractor = pcapkit.extract(fin=path, nofile=True, store=True, engine=engine) + self.addCleanup(close_extractor, extractor) + return extractor + + def _read_back(self, frames: 'tuple[bytes, ...]'): + """Write ``frames`` to a capture and read them back through the engine.""" + path = write_pcap(frames) + self.addCleanup(os.remove, path) + read = self._extract(path).frame + self.assertEqual(len(read), len(frames)) + return read + + def _frames(self): + return self._read_back(tuple(frame for _, frame, _ in FRAMES)) + + def test_ipv4_record_is_the_wire_datagram(self) -> None: + from pcapkit.toolkit import dpkt as toolkit + + for frame, (label, _, wire) in zip(self._frames()[:2], FRAMES[:2]): + with self.subTest(frame=label): + data = toolkit.ipv4_reassembly(frame, 0.0) + self.assertEqual(bytes(data.header), wire[:20]) + self.assertEqual(bytes(data.payload), wire[20:]) + # the zeroed checksums are the capture's, not ones DPKT computed + self.assertEqual(frame.ip.sum, 0) + self.assertEqual(frame.ip.data.sum, 0) + + def test_ipv6_record_is_the_wire_fragment(self) -> None: + from pcapkit.toolkit import dpkt as toolkit + + wire = FRAMES[2][1][len(ETH_IPV6):] + data = toolkit.ipv6_reassembly(self._frames()[2], 0.0) + self.assertEqual(bytes(data.header), wire[:40]) + self.assertEqual(bytes(data.payload), wire[48:]) + + def test_tcp_records_carry_the_wire_header(self) -> None: + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import dpkt as toolkit + + frames = self._frames() + for index, segment in ((1, IPV4_TCP[20:]), (3, TCP_SEGMENT)): + frame = frames[index] + for data in (toolkit.tcp_reassembly(frame, 0.0), + toolkit.tcp_traceflow(frame, 0.0, data_link=LinkType.ETHERNET)): + with self.subTest(record=type(data).__name__, frame=FRAMES[index][0]): + self.assertEqual(bytes(data.header), segment[:20]) + self.assertEqual(bytes(data.payload), segment[20:]) + + def test_tcp_header_is_found_behind_every_extension_header(self) -> None: + """One TCP segment per extension header DPKT can parse. + + The Authentication header and the odd-length Routing header are the two + whose ``len()`` disagrees with the wire; before the offset was taken from + ``length`` the first returned an empty header and the second returned the + Routing header's own trailing octets. + + """ + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import dpkt as toolkit + + frames = self._read_back(tuple(frame for _, frame in EXT_HDR_FRAMES)) + for frame, (label, _) in zip(frames, EXT_HDR_FRAMES): + with self.subTest(ext_hdr=label): + for data in (toolkit.tcp_reassembly(frame, 0.0), + toolkit.tcp_traceflow(frame, 0.0, data_link=LinkType.ETHERNET)): + self.assertEqual(bytes(data.header), TCP_SEGMENT[:20]) + self.assertEqual(bytes(data.payload), TCP_SEGMENT[20:]) + + def test_ipv6_fragment_behind_an_authentication_header(self) -> None: + """``ipv6_hdr_len`` counted a fixed list of types and missed this one.""" + from pcapkit.toolkit import dpkt as toolkit + + frame = self._read_back((AH_FRAG_FRAME,))[0] + wire = AH_FRAG_FRAME[len(ETH_IPV6):] + + self.assertEqual(toolkit.ipv6_hdr_len(frame.ip6), AH_FRAG_IHL) + data = toolkit.ipv6_reassembly(frame, 0.0) + self.assertEqual(data.ihl, AH_FRAG_IHL) + self.assertEqual(bytes(data.header), wire[:AH_FRAG_IHL]) + self.assertEqual(bytes(data.payload), TCP_SEGMENT) + + def test_packet2dict_reports_the_captured_frame(self) -> None: + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import dpkt as toolkit + + for frame, (label, wire, _) in zip(self._frames(), FRAMES): + with self.subTest(frame=label): + info = toolkit.packet2dict(frame, 0.0, data_link=LinkType.ETHERNET) + self.assertEqual(info['packet'], wire) + + def test_no_adapter_changes_what_the_next_one_sees(self) -> None: + """Every adapter, run twice over each frame, leaves it as captured.""" + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import dpkt as toolkit + + for frame, (label, wire, _) in zip(self._frames(), FRAMES): + for _ in range(2): + for adapter in (toolkit.ipv4_reassembly, toolkit.ipv6_reassembly, + toolkit.tcp_reassembly): + adapter(frame, 0.0) + toolkit.tcp_traceflow(frame, 0.0, data_link=LinkType.ETHERNET) + with self.subTest(frame=label): + # every frame here has its checksums zeroed on the wire, which is + # exactly what DPKT's own serialisation would have filled in + ip = frame.data + self.assertEqual(getattr(ip, 'sum', 0), 0) + self.assertEqual(getattr(ip.data, 'sum', 0), 0) + # and the frame still reports the octets it was read from + info = toolkit.packet2dict(frame, 0.0, data_link=LinkType.ETHERNET) + self.assertEqual(info['packet'], wire) + + def test_options_transport_matches_the_default_engine(self) -> None: + """Before the fix, 33 of this capture's IPv4 datagrams came back rewritten. + + The number of datagrams is counted here rather than pinned: the capture is + generated by ``make samples``, and that is not currently reproducible -- + the same commit produces a 43-frame and a 44-frame file in different + worktrees -- so any absolute count is flaky whichever value it picks. + + """ + from pcapkit.toolkit import dpkt as toolkit + from pcapkit.toolkit import pcap as toolkit_pcap + + try: + path = sample_path('options-transport.pcap') + except FileNotFoundError as exc: + self.skipTest(str(exc)) + + frames = self._extract(path).frame + defaults = self._extract(path, engine='default').frame + self.assertEqual(len(frames), len(defaults)) + + compared = 0 + for index, (frame, default) in enumerate(zip(frames, defaults), start=1): + expected = toolkit_pcap.ipv4_reassembly(default) + data = toolkit.ipv4_reassembly(frame, 0.0) + self.assertEqual(data is None, expected is None, index) + if data is None: + continue + compared += 1 + with self.subTest(frame=index): + self.assertEqual(bytes(data.header), bytes(expected.header)) + self.assertEqual(bytes(data.payload), bytes(expected.payload)) + self.assertEqual(data.ihl, expected.ihl) + self.assertEqual(data.tl, expected.tl) + self.assertGreater(compared, 0, 'no IPv4 datagram was compared') + + +@unittest.skipUnless(HAS_RUNTIME and HAS_DPKT, 'runtime dependencies not installed') +class DPKTUnbufferedFallbackTests(unittest.TestCase): + """What the adapters return for a packet that never came through the engine. + + A packet built by hand carries no record octets, so there is nothing to slice + and the adapters fall back to DPKT's re-serialisation. That fallback differs + from the wire -- which is the whole of #1297 -- so what it returns is pinned + here rather than left to be discovered. Unlike the class above, these tests + name :func:`~pcapkit.toolkit.dpkt.attach_buffer` and + :func:`~pcapkit.toolkit.dpkt.packet2bytes` directly, so they are a test of + that API rather than of the adapters' behaviour. + + """ + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_a_hand_built_packet_is_not_mutated(self) -> None: + import dpkt + + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import dpkt as toolkit + + packet = dpkt.ethernet.Ethernet(ETH_IPV4 + IPV4_TCP) + toolkit.packet2dict(packet, 0.0, data_link=LinkType.ETHERNET) + toolkit.ipv4_reassembly(packet, 0.0) + toolkit.tcp_reassembly(packet, 0.0) + toolkit.tcp_traceflow(packet, 0.0, data_link=LinkType.ETHERNET) + self.assertEqual(packet.ip.sum, 0) + self.assertEqual(packet.ip.tcp.sum, 0) + + def test_the_unbuffered_fallback_returns_dpkts_serialisation(self) -> None: + """Which differs from the wire in exactly the fields DPKT recomputes.""" + import dpkt + + from pcapkit.toolkit import dpkt as toolkit + + data = toolkit.ipv4_reassembly(dpkt.ethernet.Ethernet(ETH_IPV4 + IPV4_TCP), 0.0) + self.assertEqual(bytes(data.header) + bytes(data.payload), IPV4_TCP_REPACKED) + self.assertNotEqual(IPV4_TCP_REPACKED, IPV4_TCP) + + # the TCP layer alone round-trips: dpkt.tcp.TCP.__bytes__ recomputes + # nothing, the pseudo-header checksum being dpkt.ip.IP.__bytes__'s doing + tcp = toolkit.tcp_reassembly(dpkt.ethernet.Ethernet(ETH_IPV4 + IPV4_TCP), 0.0) + self.assertEqual(bytes(tcp.header), IPV4_TCP[20:40]) + + def test_attaching_the_buffer_makes_the_octets_the_wires_again(self) -> None: + import dpkt + + from pcapkit.toolkit import dpkt as toolkit + + frame = ETH_IPV4 + IPV4_TCP + packet = dpkt.ethernet.Ethernet(frame) + toolkit.attach_buffer(packet, frame) + self.assertEqual(toolkit.packet2bytes(packet), frame) + data = toolkit.ipv4_reassembly(packet, 0.0) + self.assertEqual(bytes(data.header) + bytes(data.payload), IPV4_TCP) + + def test_a_misplaced_slice_is_refused_rather_than_returned(self) -> None: + """A buffer that is not the packet's own cannot be sliced into.""" + import dpkt + + from pcapkit.toolkit import dpkt as toolkit + from pcapkit.utilities.exceptions import ProtocolError + + frame = ETH_IPV4 + IPV4_TCP + packet = dpkt.ethernet.Ethernet(frame) + # one octet short at the front, so every layer sits one octet early + toolkit.attach_buffer(packet, frame[1:]) + with self.assertRaises(ProtocolError): + toolkit.ipv4_reassembly(packet, 0.0) + + +if __name__ == '__main__': + unittest.main()