From 9a9d31793d5566021306c08f5315bdb9cf458651 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 13 Oct 2026 13:26:58 -0400 Subject: [PATCH 1/4] fix(toolkit): dpkt walks IP layers off the octets as the default engine does (#1609, #1626, #1651, #1654) The dpkt adapters walked what dpkt decoded, which is not what the default engine dissects. They now read every IP layer, extension header and TCP segment off the record octets, by the default engine's rules: - #1651: an IPv6 payload ends at its Payload Length, or at a jumbogram's Jumbo Payload Length; a zero one with no Jumbo Payload option has none. - #1654: IP behind a VLAN tag stack dpkt leaves undecoded is read too. - #1609: TCP and IP inputs stop at 32 extension headers in a row and at the 64-layer frame bound. - #1626: Mobility, HIP and Shim6 are walked, and so is a Fragment header with another header behind it, in a frame dpkt cannot decode as well, which is read by its link type (attach_linktype). - packet2dict maps a frame kept raw to {} rather than raising. Closes #1651 Closes #1654 Part of #1609, #1626 --- pcapkit/toolkit/dpkt.py | 1126 +++++++++-------- ...t_dpkt_ip_walk_1609_1626_1651_1654_unit.py | 365 ++++++ .../test_dpkt_reassembly_rawbytes_unit.py | 14 +- tests/toolkit/test_dpkt_unit.py | 67 +- 4 files changed, 985 insertions(+), 587 deletions(-) create mode 100644 tests/toolkit/test_dpkt_ip_walk_1609_1626_1651_1654_unit.py diff --git a/pcapkit/toolkit/dpkt.py b/pcapkit/toolkit/dpkt.py index f4e0db2485..a531bcaa5c 100644 --- a/pcapkit/toolkit/dpkt.py +++ b/pcapkit/toolkit/dpkt.py @@ -15,35 +15,39 @@ import copy import decimal import ipaddress -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, NamedTuple, cast +from pcapkit.const.reg.ethertype import EtherType as Enum_EtherType +from pcapkit.const.reg.linktype import LinkType as Enum_LinkType from pcapkit.const.reg.transtype import TransType as Enum_TransType from pcapkit.foundation.reassembly.data.ip import Packet as IP_Packet from pcapkit.foundation.reassembly.data.tcp import Packet as TCP_Packet from pcapkit.foundation.traceflow.data.data import FrameRecord from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet from pcapkit.protocols.data.misc.pcap.frame import FrameInfo as Data_FrameInfo -from pcapkit.toolkit.pypcapfile import (IPV4_EXTENSION_HEADERS, TCP_MIN_HEADER_LEN, VLAN_TAG_LEN, - VLAN_TAGS, _default_accepts, _past_extension_headers, +from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT +from pcapkit.protocols.link.loopback import family_byteorder +from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT +from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length +from pcapkit.toolkit.pypcapfile import (IPV4_EXTENSION_HEADERS, IPV4_HEADER_LEN, + IPV6_EXTENSION_HEADERS, IPV6_HEADER_LEN, TCP_MIN_HEADER_LEN, + VLAN_TAG_LEN, VLAN_TAGS, _default_accepts, _walk_extension_headers) from pcapkit.utilities.exceptions import ProtocolError, UnsupportedCall if TYPE_CHECKING: from decimal import Decimal from ipaddress import IPv4Address, IPv6Address - from typing import Any, Optional + from typing import Any, Iterator, Optional from dpkt.dpkt import Packet from dpkt.ethernet import Ethernet from dpkt.ip import IP - from dpkt.ip6 import IP6, IP6FragmentHeader - from dpkt.tcp import TCP - - from pcapkit.const.reg.linktype import LinkType as Enum_LinkType + from dpkt.ip6 import IP6 __all__ = [ 'RecordTimestamp', 'Timestamp', 'DecimalTimestamp', 'ipv6_hdr_len', 'attach_timestamp', 'packet2timestamp', 'attach_buffer', 'packet2bytes', - 'packet2frame', 'packet2chain', 'packet2dict', + 'attach_linktype', 'packet2frame', 'packet2chain', 'packet2dict', 'ipv4_reassembly', 'ipv6_reassembly', 'tcp_reassembly', 'tcp_traceflow' ] @@ -76,18 +80,40 @@ #: .. _DPKT: https://dpkt.readthedocs.io BUFFER_ATTR = '__pcapkit_buffer__' +#: Attribute a frame's data link type is stashed under, as the capture's +#: ``LINKTYPE_*`` number, by :func:`attach_linktype`. +LINKTYPE_ATTR = '__pcapkit_linktype__' + #: IP protocol numbers of the IP-in-IP tunnels the default engine dissects #: (see :attr:`Internet.__proto__ #: `), over IPv4 and #: IPv6 alike: IPv4 (protocol 4, :rfc:`2003`) and IPv6 (protocol 41, #: :rfc:`2473` and :rfc:`4213`). `DPKT`_ also decodes an IPv4 packet's #: protocol 0 as IPv4, its ``IP_PROTO_IP``, where the default engine reads an -#: IPv6 Hop-by-Hop Options header, so neither :func:`_tcp_carrier` nor -#: :func:`_first_ipv6` enters it. +#: IPv6 Hop-by-Hop Options header, so :func:`_ip_layers` does not enter it. #: #: .. _DPKT: https://dpkt.readthedocs.io IP_TUNNELS = frozenset((int(Enum_TransType.IPv4), int(Enum_TransType.IPv6))) +#: Length of an Ethernet II header: two MAC addresses and the EtherType. +ETHERNET_HEADER_LEN = 14 + +#: Length of a BSD loopback header: the address family. +LOOPBACK_HEADER_LEN = 4 + +#: EtherTypes of the IP layers the default engine dissects over Ethernet, each +#: mapped to its IP version (see :attr:`Link.__proto__ +#: `). +ETHERTYPE_IP_VERSIONS = { + Enum_EtherType.Internet_Protocol_version_4: 4, + Enum_EtherType.Internet_Protocol_version_6: 6, +} # type: dict[int, int] + +#: Address families of the IP layers the default engine dissects over BSD +#: loopback, each mapped to its IP version (see :attr:`Loopback.__proto__ +#: `). +LOOPBACK_IP_VERSIONS = {2: 4, 24: 6, 28: 6, 30: 6} # type: dict[int, int] + #: Destination address prefixes by which `DPKT`_ reads an Ethernet frame whose #: EtherType field is an 802.3 length, at most 1500, as Cisco ISL; the default #: engine reads one as raw octets (see :func:`_ethernet_accepted`). @@ -260,6 +286,26 @@ def attach_buffer(packet: 'Packet', buffer: 'bytes') -> 'None': setattr(packet, BUFFER_ATTR, bytes(buffer)) +def attach_linktype(packet: 'Packet', linktype: 'int') -> 'None': + """Stash the data link type of a frame's capture on the frame. + + Args: + packet: DPKT packet. + linktype: ``LINKTYPE_*`` number: the PCAP global header's, or the + PCAP-NG interface's the packet was captured on. + + Note: + A frame `DPKT`_ cannot decode is kept as raw octets (see + :meth:`DPKT.read_frame `), + which do not say what link layer they start with. The adapters read its + layers off those octets by this (see :func:`_link_network`). + + .. _DPKT: https://dpkt.readthedocs.io + + """ + setattr(packet, LINKTYPE_ATTR, int(linktype)) + + def packet2bytes(packet: 'Packet') -> 'bytes': """Fetch the octets of a DPKT packet without modifying it. @@ -399,68 +445,27 @@ def _vlan_tpids(wire: 'bytes | memoryview') -> 'list[int]': return tpids -def _wire_hdr_len(packet: 'Packet', payload: 'Packet', wire: 'Optional[bytes | memoryview]' = None) -> 'int': - """Calculate the octets a layer occupies before its payload begins. - - Args: - packet: DPKT packet, i.e. one layer of a frame. - payload: That layer's payload, i.e. :attr:`packet.data `. - wire: The captured octets of ``packet``, from its first on, if any. - - Returns: - The advance :meth:`~dpkt.dpkt.Packet.unpack` itself made to reach the - payload. - - For IPv6 that is the fixed header plus each extension header's - :attr:`~dpkt.ip6.IP6ExtensionHeader.length`, which is what - :meth:`dpkt.ip6.IP6.unpack` walks the buffer by. It cannot be taken from - ``len()``: :meth:`dpkt.ip6.IP6.__len__` sums ``len()`` of the extension - headers instead, and that is 12 octets plus the *rest of the frame* for - an Authentication header -- :class:`dpkt.ip6.IP6AHHeader` trims only into - :attr:`~dpkt.ip6.IP6AHHeader.auth_data` and leaves - :attr:`~dpkt.dpkt.Packet.data` running to the end of the buffer -- and 8 - octets short of the wire for a Routing header whose Hdr Ext Len is odd - (:class:`dpkt.ip6.IP6RoutingHeader` keeps whole addresses only). - - Every other layer's ``len()`` does bound its payload, so the difference - of the two is used there. For IPv4 that difference is the Internet - Header Length, options included. For an Ethernet II frame whose - ``wire`` is given, 4 octets are added for each VLAN tag - :class:`dpkt.ethernet.Ethernet` stepped over but dropped from its - :attr:`~dpkt.ethernet.Ethernet.vlan_tags`, which ``len()`` counts (see - :func:`_vlan_tpids`). - - """ - import dpkt # isort:skip - - if hasattr(packet, 'extension_hdrs'): - return packet.__hdr_len__ + sum( - ext_hdr.length for ext_hdr in _ipv6_ext_hdrs(cast('IP6', packet)) - ) - hdr_len = len(packet) - len(payload) - if wire is not None and isinstance(packet, dpkt.ethernet.Ethernet) and int.from_bytes(wire[12:14], 'big') > 1500: - dropped = len(_vlan_tpids(wire)) - len(getattr(packet, 'vlan_tags', None) or ()) - hdr_len += VLAN_TAG_LEN * dropped - return hdr_len - - -def _layer2bytes(packet: 'Packet', layer: 'Packet') -> 'tuple[bytes, bool]': - """Fetch the octets of a layer of a DPKT packet, from its first octet on. +def _network_octets(packet: 'Packet', ip: 'IP | IP6') -> 'bytes': + """Fetch the octets of a DPKT packet's network layer, from its first octet on. Args: packet: DPKT packet, i.e. the outermost layer. - layer: A layer nested within ``packet``. + ip: Its network layer, as :func:`_network` finds it: ``packet`` itself, + or the payload of its link layer. Returns: - The octets, and whether they are the captured ones. With a buffer - attached by :func:`attach_buffer` they are the record octets from the - layer's offset to the end of the frame, so any link-layer trailer is - still there for the caller to trim; otherwise they are the layer alone, - serialised from a copy (see :func:`packet2bytes`). + With a buffer attached by :func:`attach_buffer`, the record octets from + the layer's offset to the end of the frame, so any link-layer trailer is + still there; otherwise the layer alone, serialised from a copy (see + :func:`packet2bytes`). Behind an Ethernet II header, the offset counts + the VLAN tags off the octets, as :class:`dpkt.ethernet.Ethernet` keeps + only some of a stack it reads in two goes (see :func:`_vlan_tpids`); + behind any other link layer, it is what the layer's ``len()`` counts + before its payload. Raises: ProtocolError: If the buffer does not carry the layer's own header where - the walk above placed it. `DPKT`_ unpacks a header field by field and + the link layer placed it. `DPKT`_ unpacks a header field by field and :meth:`~dpkt.dpkt.Packet.pack_hdr` packs those fields back without recomputing any of them, so the two agree octet for octet wherever the offset is right -- which makes this an exact check rather than a @@ -471,28 +476,24 @@ def _layer2bytes(packet: 'Packet', layer: 'Packet') -> 'tuple[bytes, bool]': .. _DPKT: https://dpkt.readthedocs.io """ - buffer = getattr(packet, BUFFER_ATTR, None) - if buffer is not None: + import dpkt # isort:skip + + buffer = getattr(packet, BUFFER_ATTR, None) # type: Optional[bytes] + if buffer is None: + return packet2bytes(ip) + if ip is packet: offset = 0 - current = packet - view = memoryview(cast('bytes', buffer)) - while current is not layer: - payload = getattr(current, 'data', None) - if payload is None or isinstance(payload, (bytes, bytearray)): - break - offset += _wire_hdr_len(current, payload, view[offset:]) - current = payload - else: - wire = cast('bytes', buffer)[offset:] - hdr_len = getattr(layer, '__hdr_len__', 0) - if hdr_len and wire[:hdr_len] != layer.pack_hdr(): - raise ProtocolError( - f'{type(layer).__name__} header not found at offset {offset} of the ' - f'{len(cast("bytes", buffer))}-octet record; the frame cannot be ' - 'sliced without re-serialising it' - ) - return wire, True - return packet2bytes(layer), False + elif isinstance(packet, dpkt.ethernet.Ethernet): + offset = ETHERNET_HEADER_LEN + VLAN_TAG_LEN * len(_vlan_tpids(buffer)) + else: + offset = len(packet) - len(ip) + wire = buffer[offset:] + if wire[:ip.__hdr_len__] != ip.pack_hdr(): + raise ProtocolError( + f'{type(ip).__name__} header not found at offset {offset} of the ' + f'{len(buffer)}-octet record; the frame cannot be sliced without re-serialising it' + ) + return wire def _ethernet_accepted(packet: 'Ethernet') -> 'bool': @@ -608,6 +609,29 @@ def _network_depth(packet: 'Packet') -> 'int': return 2 + tags +def _ipv4_header_accepted(header: 'bytes') -> 'bool': + """Test if the default engine reads an IPv4 header as an IPv4 layer. + + Args: + header: The header's captured octets, from its first 20 on to as many as + its IHL counts, or to the end of the capture if that comes first. + + Returns: + :data:`False` if the default engine's IPv4 parser rejects it -- its + version is not 4, its options are ones the parser rejects, or its IHL + runs past the captured octets (:issue:`1607`) -- as the default engine + then dissects nothing past it; else :data:`True`. + + Note: + A header of version 4 with no options is not parsed, as the parser + checks nothing else of one; any other is asked of the parser itself, as + the pypcapfile engine asks it (see + :func:`pcapkit.toolkit.pypcapfile._default_accepts`). + + """ + return header[0] == 0x45 or _default_accepts('IPv4', bytes(header)) + + def _ipv4_accepted(ipv4: 'IP') -> 'bool': """Test if the default engine reads a DPKT IPv4 packet as an IPv4 layer. @@ -615,195 +639,420 @@ def _ipv4_accepted(ipv4: 'IP') -> 'bool': ipv4: DPKT IPv4 packet. Returns: - :data:`False` if the default engine's IPv4 parser rejects its header -- - its version is not 4, its options are ones the parser rejects, or its - IHL runs past the captured octets (:issue:`1607`) -- as the default - engine then dissects nothing past it; else :data:`True`. + What :func:`_ipv4_header_accepted` says of its header. `DPKT`_ decodes + what an IPv4 EtherType or tunnel carries as IPv4 whatever its version, + and never checks its options. Its octets are the fields as `DPKT`_ read + them and the options as captured, which is exact. - Note: - `DPKT`_ decodes what an IPv4 EtherType or tunnel carries as IPv4 - whatever its version, and never checks its options, so the header is - asked of the parser itself, as the pypcapfile engine asks it (see - :func:`pcapkit.toolkit.pypcapfile._default_accepts`). Its octets are the - fields as `DPKT`_ read them and the options as captured, which is - exact. A header of version 4 with no options is not parsed: `DPKT`_ - decodes none shorter than 20 octets or with an IHL below 5, and the - parser checks nothing else of one. + .. _DPKT: https://dpkt.readthedocs.io + + """ + return _ipv4_header_accepted(ipv4.pack_hdr() + bytes(ipv4.opts)) + + +class _Segment(NamedTuple): + """A TCP segment the default engine reads, keyed by the IP layer that carries it (see :func:`_tcp_carrier`).""" + + #: Source address of the IP layer. + src: 'IPv4Address | IPv6Address' + #: Destination address of the IP layer. + dst: 'IPv4Address | IPv6Address' + #: Source port. + srcport: 'int' + #: Destination port. + dstport: 'int' + #: Sequence number. + seq: 'int' + #: Acknowledgement number. + ack: 'int' + #: Reset (RST) flag. + rst: 'bool' + #: Synchronise (SYN) flag. + syn: 'bool' + #: Finish (FIN) flag. + fin: 'bool' + #: Captured header, as far as the Data Offset goes. + header: 'bytes' + #: Captured payload, as far as the IP packet carrying it goes. + payload: 'bytes' + + +class _IPLayer(NamedTuple): + """An IPv4 or IPv6 header the default engine dissects as a layer of a frame (see :func:`_ip_layer`).""" + + #: IP version of the header, ``4`` or ``6``. + version: 'int' + #: Captured octets from the header on, to the end of the frame or of the + #: packets carrying it, whichever comes first. + octets: 'bytes' + #: Place of the layer in the frame, its first layer being ``1``, as the + #: default engine counts towards + #: :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`. + depth: 'int' + + +class _Upper(NamedTuple): + """The layer the default engine dissects behind an IP header and the headers it reads past.""" + + #: Protocol number of the layer. + protocol: 'int' + #: Captured octets from its header on, to the end of the packet carrying it. + octets: 'bytes' + #: Place of the layer in the frame, as for :attr:`_IPLayer.depth`. + depth: 'int' + #: Whether an AH or IPv6 extension header in IPv4 comes before it, which the + #: default engine dissects as a layer of its own (see :func:`_tcp_carrier`). + behind: 'bool' + + +def _ip_layer(version: 'int', octets: 'bytes', depth: 'int') -> 'Optional[_IPLayer]': + """Read an IP header as the default engine dissects it. + + Args: + version: IP version to read the header as, ``4`` or ``6``. + octets: Captured octets from the header on, to the end of the frame or + of the packets carrying it. + depth: Place of the header in the frame (see :attr:`_IPLayer.depth`). + + Returns: + The layer, or :data:`None` if the default engine dissects none there: + past :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers + (:issue:`1610`), short of the fixed header, or an IPv4 header whose IHL + is below 5 or which its parser rejects (see + :func:`_ipv4_header_accepted`). `DPKT`_ decodes no header that short or + with that IHL either, and neither it nor the default engine checks an + IPv6 header's version. .. _DPKT: https://dpkt.readthedocs.io """ - if ipv4.v == 4 and ipv4.hl == 5: - return True - return _default_accepts('IPv4', ipv4.pack_hdr() + bytes(ipv4.opts)) + if depth > FRAME_LAYER_LIMIT: + return None + if version == 4: + ihl = (octets[0] & 0x0F) * 4 if len(octets) >= IPV4_HEADER_LEN else 0 + if ihl < IPV4_HEADER_LEN or not _ipv4_header_accepted(octets[:ihl]): + return None + elif len(octets) < IPV6_HEADER_LEN: + return None + return _IPLayer(version, bytes(octets), depth) -def _past_ipv4_extension_headers(packet: 'Packet', ipv4: 'IP') -> 'Optional[tuple[int, bytes]]': - """Step over the headers at the head of a DPKT IPv4 packet's payload that the default engine reads past. +def _link_network(linktype: 'Optional[int]', octets: 'bytes') -> 'Optional[_IPLayer]': + """Find the IP layer the default engine dissects right after a frame's link layer, from its octets. Args: - packet: DPKT packet ``ipv4`` is a layer of, whose octets - :func:`_layer2bytes` slices. - ipv4: DPKT IPv4 packet, whose protocol is one of the - :data:`~pcapkit.toolkit.pypcapfile.IPV4_EXTENSION_HEADERS`. + linktype: The frame's ``LINKTYPE_*`` number (see :func:`attach_linktype`), + if known. + octets: The frame's record octets. Returns: - The protocol number after the last of those headers and the octets from - its header on, as far as the Total Length goes -- a Total Length of - ``0``, which TCP segmentation offload leaves, runs to the end of the - frame -- or :data:`None` if the default engine reads no upper-layer - header there: ``ipv4`` is a later fragment, whose data is a slice of the - datagram's payload (:rfc:`791#section-3.2`), or it leaves one of the - headers undissected, which - :func:`~pcapkit.toolkit.pypcapfile._past_extension_headers` decides by - the default engine's own rules, up to - :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` - headers in a row (:issue:`1604`). - - Note: - The default engine dispatches an IPv4 packet's protocol through the - registry IPv6 uses, so it reads an AH or IPv6 extension header there - and what its Next Header names. `DPKT`_ decodes protocol 0 as IPv4 and - an AH as :class:`dpkt.ah.AH`, and keeps the rest as octets, so the - octets are read instead (:issue:`1606`). - - .. _DPKT: https://dpkt.readthedocs.io + The layer (see :func:`_ip_layer`), or :data:`None` if there is none. + Over Ethernet II, that is an IPv4 or IPv6 EtherType after the 14-octet + header and every 802.1Q and 802.1ad tag behind it (the + :data:`~pcapkit.toolkit.pypcapfile.VLAN_TAGS`), each tag a layer of its + own; over BSD loopback (``NULL`` and ``LOOP``), an IPv4 or IPv6 address + family (:data:`LOOPBACK_IP_VERSIONS`) in the byte order + :func:`~pcapkit.protocols.link.loopback.family_byteorder` gives. An + ``IPV4`` or ``IPV6`` frame is IP of that version, and a ``RAW`` one of + the version its first octet gives. The default engine leaves every other + link type, EtherType and family raw, and a frame cut short of its link + header. """ - if ipv4.offset: + if linktype == Enum_LinkType.ETHERNET: + if len(octets) < ETHERNET_HEADER_LEN: + return None + ethertype, start = int.from_bytes(octets[12:14], 'big'), ETHERNET_HEADER_LEN + while ethertype in VLAN_TAGS: + if len(octets) < start + VLAN_TAG_LEN: + return None + ethertype, start = int.from_bytes(octets[start + 2:start + 4], 'big'), start + VLAN_TAG_LEN + version = ETHERTYPE_IP_VERSIONS.get(ethertype) + depth = 2 + (start - ETHERNET_HEADER_LEN) // VLAN_TAG_LEN + elif linktype in (Enum_LinkType.NULL, Enum_LinkType.LOOP): + if len(octets) < LOOPBACK_HEADER_LEN: + return None + family, start, depth = octets[:LOOPBACK_HEADER_LEN], LOOPBACK_HEADER_LEN, 2 + version = LOOPBACK_IP_VERSIONS.get(int.from_bytes(family, family_byteorder(family, linktype))) + elif linktype in (Enum_LinkType.IPV4, Enum_LinkType.IPV6, Enum_LinkType.RAW) and octets: + version = {Enum_LinkType.IPV4: 4, Enum_LinkType.IPV6: 6}.get(linktype, octets[0] >> 4) + start, depth = 0, 1 + else: return None - wire, _ = _layer2bytes(packet, ipv4) - ihl = ipv4.hl * 4 - return _past_extension_headers(ipv4.p, wire[ihl:max(ipv4.len, ihl)] if ipv4.len else wire[ihl:]) + if version not in (4, 6): + return None + return _ip_layer(version, octets[start:], depth) -def _decode_tcp(segment: 'bytes') -> 'Optional[TCP]': - """Decode a TCP segment the default engine reads behind an AH or IPv6 extension header in IPv4. +def _network_layer(packet: 'Packet') -> 'Optional[_IPLayer]': + """Fetch the network layer of a DPKT packet, as the default engine dissects it. Args: - segment: Raw octets from the TCP header on, as far as the IPv4 packet - carrying it goes (see :func:`_past_ipv4_extension_headers`). + packet: DPKT packet, i.e. the outermost layer. Returns: - The segment, with ``segment`` attached by :func:`attach_buffer` so that - its header is the captured one; or :data:`None` if the default engine - reads no TCP there: fewer than :data:`TCP_MIN_HEADER_LEN - ` octets, a Data Offset - counting fewer or more than were captured, or options its TCP parser - rejects (see :func:`~pcapkit.toolkit.pypcapfile._default_accepts`). + The IPv4 or IPv6 packet :func:`_network` finds, with the octets + :func:`_network_octets` slices for it and its place in the frame as + :func:`_network_depth` counts it; or, if `DPKT`_ left the frame's + payload undecoded, the layer :func:`_link_network` reads off the record + octets :func:`attach_buffer` attached. That is a frame behind a VLAN tag + stack `DPKT`_ does not finish decoding in two passes (:issue:`1654`), + an IPv6 packet whose extension headers it cannot decode, and a frame it + cannot decode at all, which the engine keeps as raw octets with its link + type attached by :func:`attach_linktype` (:issue:`1626`). Else + :data:`None`. Note: - Directly over IP, the default engine reads a TCP header its parser - rejects out of the raw octets left behind - (:func:`pcapkit.toolkit.pcap.tcp_segment`), but behind another header - it does not, so the parser's answer stands here, as for the pypcapfile - engine. + A packet built by hand carries no record octets, so its network layer's + octets are `DPKT`_'s re-serialisation of it (see :func:`packet2bytes`), + and one that is no :class:`dpkt.ip.IP` or :class:`dpkt.ip6.IP6` has + none. + + .. _DPKT: https://dpkt.readthedocs.io """ import dpkt # isort:skip - hdr_len = (segment[12] >> 4) * 4 if len(segment) >= TCP_MIN_HEADER_LEN else 0 - if hdr_len < TCP_MIN_HEADER_LEN or hdr_len > len(segment): + ip = _network(packet) + if ip is not None: + if not isinstance(ip, (dpkt.ip.IP, dpkt.ip6.IP6)): + return None + return _ip_layer(6 if isinstance(ip, dpkt.ip6.IP6) else 4, _network_octets(packet, ip), + _network_depth(packet)) + + buffer = getattr(packet, BUFFER_ATTR, None) # type: Optional[bytes] + if buffer is None or not isinstance(getattr(packet, 'data', None), (bytes, bytearray)): + return None + if isinstance(packet, dpkt.ethernet.Ethernet): + return _link_network(Enum_LinkType.ETHERNET, buffer) + if isinstance(packet, dpkt.loopback.Loopback): # an address family it has no decoder for return None - if hdr_len > TCP_MIN_HEADER_LEN and not _default_accepts('TCP', segment[:hdr_len]): + return _link_network(getattr(packet, LINKTYPE_ATTR, None), buffer) + + +def _ipv4_upper(layer: '_IPLayer') -> 'Optional[_Upper]': + """Step over an IPv4 header to the layer the default engine dissects behind it. + + Args: + layer: IPv4 layer. + + Returns: + The layer its protocol names, in the octets its Total Length leaves -- + a Total Length of ``0``, which TCP segmentation offload leaves, the rest + of what carries it -- past the headers the default engine reads past, + one layer each (see + :func:`~pcapkit.toolkit.pypcapfile._walk_extension_headers`); or + :data:`None` if it dissects none: in a later fragment, whose data is a + slice of the datagram's payload (:rfc:`791#section-3.2`), or behind one + of those headers it leaves undissected. + + """ + octets = layer.octets + if int.from_bytes(octets[6:8], 'big') & 0x1FFF: # a later fragment + return None + ihl, total = (octets[0] & 0x0F) * 4, int.from_bytes(octets[2:4], 'big') + payload = octets[ihl:max(total, ihl)] if total else octets[ihl:] + if octets[9] not in IPV4_EXTENSION_HEADERS: + return _Upper(octets[9], payload, layer.depth + 1, False) + walked = _walk_extension_headers(octets[9], payload) + if walked is None: return None - tcp = dpkt.tcp.TCP(segment) - attach_buffer(tcp, segment) - return tcp + protocol, upper, hops = walked + return _Upper(protocol, upper, layer.depth + hops + 1, True) -def _decode_ip(protocol: 'int', data: 'bytes') -> 'Optional[IP | IP6]': - """Decode an IP packet the default engine reads behind an AH or IPv6 extension header in IPv4. +def _ipv6_chain(layer: '_IPLayer') -> 'tuple[bytes, Optional[int], Optional[_Upper]]': + """Walk an IPv6 packet's extension headers as the default engine does. Args: - protocol: The IP-in-IP protocol number, one of the :data:`IP_TUNNELS`. - data: Raw octets from the IP header on, as far as the IPv4 packet - carrying it goes (see :func:`_past_ipv4_extension_headers`). + layer: IPv6 layer. Returns: - The packet as `DPKT`_ decodes one in a tunnel, with ``data`` attached by - :func:`attach_buffer`, so that :func:`_layer2bytes` slices its layers - out of the captured octets; or :data:`None` if decoding it raises - anything at all, as :class:`~pcapkit.foundation.engines.dpkt.DPKT` then - keeps a frame raw. `DPKT`_ raises more than its own - :exc:`~dpkt.dpkt.UnpackError`: dpkt 1.9.8 reads ``frag_off`` from - whatever header follows an IPv6 Fragment header, so one followed by - another extension header raises :exc:`AttributeError` (:issue:`1351`). + The packet's octets as far as its Payload Length goes -- for a zero one, + the Jumbo Payload Length (:rfc:`2675`) of a Hop-by-Hop Options header + right after the IPv6 header, and lacking one, no payload at all + (:issue:`1651`) -- then where in them its first Fragment header starts, + if the walk reaches one, and the layer behind the last header walked, or + :data:`None` if the default engine dissects none. + + Each of the :data:`~pcapkit.toolkit.pypcapfile.IPV6_EXTENSION_HEADERS` + is walked, Mobility, HIP and Shim6 included (:issue:`1626`), and is a + layer of the frame: an AH is ``(Payload Length + 2) * 4`` octets long, a + Fragment header 8, and any other ``(Hdr Ext Len + 1) * 8``. The walk + ends with no layer behind past + :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` + headers (:issue:`1604`) or + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the + frame (:issue:`1610`), at a header cut short or overrunning the packet, + and at a later fragment's Fragment header, behind which lies a slice of + the Fragmentable Part (:rfc:`8200#section-4.5`). + + Note: + Where a header's own parser rejects it, the default engine reads it + with a generic one that takes the same length, so the lengths alone + decide its walk. `DPKT`_'s walk does not stop at either bound, decodes + none of 135, 139 and 140, runs a zero Payload Length to the end of the + frame, and raises :exc:`AttributeError` at a Fragment header first in + the chain with another header behind it (:issue:`1351`), which keeps the + whole frame raw, so the walk is the default engine's own. .. _DPKT: https://dpkt.readthedocs.io """ - import dpkt # isort:skip + octets = layer.octets + extent = (int.from_bytes(octets[4:6], 'big') + or jumbo_payload_length(octets[6], octets[IPV6_HEADER_LEN:]) or 0) + data = octets[:IPV6_HEADER_LEN + extent] + nxt, offset, count, fragment = octets[6], IPV6_HEADER_LEN, 0, None # type: int, int, int, Optional[int] + while nxt in IPV6_EXTENSION_HEADERS: + if (count == EXTENSION_HEADER_LIMIT or layer.depth + count >= FRAME_LAYER_LIMIT + or len(data) < offset + 8): + return data, fragment, None + if nxt == Enum_TransType.IPv6_Frag: + if fragment is None: + fragment = offset + if int.from_bytes(data[offset + 2:offset + 4], 'big') >> 3: # a later fragment + return data, fragment, None + length = 8 + elif nxt == Enum_TransType.AH: + length = (data[offset + 1] + 2) * 4 + else: + length = (data[offset + 1] + 1) * 8 + nxt, offset, count = data[offset], offset + length, count + 1 + if len(data) < offset: + return data, fragment, None + return data, fragment, _Upper(nxt, data[offset:], layer.depth + count + 1, False) - klass = dpkt.ip.IP if protocol == Enum_TransType.IPv4 else dpkt.ip6.IP6 - try: - ip = klass(data) - except Exception: # as the DPKT engine does, for a whole frame - return None - attach_buffer(ip, data) - return ip +def _ip_layers(packet: 'Packet') -> 'Iterator[tuple[_IPLayer, Optional[_Upper]]]': + """Walk the IP layers of a DPKT packet that the default engine dissects. + + Args: + packet: DPKT packet, i.e. the outermost layer. -def _tcp_carrier(packet: 'Packet') -> 'tuple[Optional[IP | IP6], Optional[TCP]]': - """Fetch the IP layer of a DPKT packet that carries its TCP segment, and the segment. + Yields: + Each IP layer, outermost first, with the layer the default engine + dissects behind it, if any (see :func:`_ipv4_upper` and + :func:`_ipv6_chain`). The first is the network layer (see + :func:`_network_layer`), and each next one the IPv4 or IPv6 packet the + last carries in an IP-in-IP tunnel (the :data:`IP_TUNNELS`), as deep as + they go. + + Note: + Every layer is read off the octets, by the default engine's rules, + rather than off what `DPKT`_ decoded of them, which differs from it in + the ways :func:`_ipv6_chain` and :func:`_network_layer` give. + + .. _DPKT: https://dpkt.readthedocs.io + + """ + layer = _network_layer(packet) + while layer is not None: + upper = _ipv4_upper(layer) if layer.version == 4 else _ipv6_chain(layer)[2] + yield layer, upper + if upper is None or upper.protocol not in IP_TUNNELS: + return + layer = _ip_layer(4 if upper.protocol == Enum_TransType.IPv4 else 6, upper.octets, upper.depth) + + +def _first_layer(packet: 'Packet', version: 'int') -> 'Optional[_IPLayer]': + """Fetch the first IP layer of a version in a DPKT packet, as the default engine dissects it. Args: packet: DPKT packet, i.e. the outermost layer. + version: IP version, ``4`` or ``6``. Returns: - The innermost IPv4 or IPv6 packet of the network layer (see - :func:`_network`) and of the IP-in-IP tunnels it carries, and its - :class:`dpkt.tcp.TCP` payload, if it has one; else :data:`None` for the - latter, and for both with no network layer, or if the default engine - rejects the header of an IPv4 packet on the way (see - :func:`_ipv4_accepted`). A tunnel is a packet whose protocol, or for - IPv6 the next header after its extension headers, is one of the - :data:`IP_TUNNELS`. An IPv4 packet whose protocol names an AH or IPv6 - extension header carries what the default engine reads past them (see - :func:`_past_ipv4_extension_headers`), a tunnel or a TCP segment, which - are decoded from the captured octets (see :func:`_decode_ip` and - :func:`_decode_tcp`). + The first such layer :func:`_ip_layers` walks, or :data:`None`. Note: - `DPKT`_ decodes an IPv4 or IPv6 packet inside another (:rfc:`2003`, - :rfc:`2473`, :rfc:`4213`) as its ``data``. The TCP endpoints are the - innermost packet's addresses, as for the default engine - (:func:`pcapkit.toolkit.pcap.tcp_segment`); the outer ones are the - tunnel's (:issue:`1581`). A segment found behind an extension header is - no layer of ``packet``, so it carries its own octets, which - :func:`_layer2bytes` then hands back (:issue:`1606`). + The default engine hands IP reassembly ``frame['IPv4']`` or + ``frame['IPv6']``, the first such layer however deep + (:func:`pcapkit.toolkit.pcap.ipv4_reassembly`), so IPv4 carried in IPv6 + counts, and IPv6 in IPv4 (:issue:`1615`, :issue:`1637`). + + """ + return next((layer for layer, _ in _ip_layers(packet) if layer.version == version), None) + + +def _tcp_carrier(packet: 'Packet') -> 'Optional[_Segment]': + """Fetch the TCP segment of a DPKT packet, keyed by the IP layer that carries it. + + Args: + packet: DPKT packet, i.e. the outermost layer. + + Returns: + The segment behind the innermost IP layer :func:`_ip_layers` walks, if + the default engine reads one there, inside + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the + frame (:issue:`1609`); else :data:`None`. It has at least + :data:`TCP_MIN_HEADER_LEN ` + octets, and a Data Offset counting no fewer; behind an AH or IPv6 + extension header in IPv4, also no more octets than were captured, and + options its TCP parser accepts (see + :func:`~pcapkit.toolkit.pypcapfile._default_accepts`). + + Note: + Directly over IP, the default engine reads a TCP header its parser + rejects out of the raw octets left behind + (:func:`pcapkit.toolkit.pcap.tcp_segment`), as `DPKT`_ decodes one, and + an IPv6 packet's extension headers do not come between, as it dissects + them itself. Behind a header that is a layer of its own, it does not, so + the parser's answer stands there, as for the pypcapfile engine. The TCP + endpoints are the innermost IP layer's addresses, as for the default + engine; the outer ones are the tunnel's (:issue:`1581`). Each flag is + read from the flags octet alone, as the default engine reads it + (:issue:`1618`). .. _DPKT: https://dpkt.readthedocs.io """ - import dpkt # isort:skip + walked = None + for walked in _ip_layers(packet): + pass + if walked is None: + return None + layer, upper = walked + if upper is None or upper.protocol != Enum_TransType.TCP or upper.depth > FRAME_LAYER_LIMIT: + return None + segment = upper.octets + hdr_len = (segment[12] >> 4) * 4 if len(segment) >= TCP_MIN_HEADER_LEN else 0 + if hdr_len < TCP_MIN_HEADER_LEN: + return None + if upper.behind and (hdr_len > len(segment) or ( + hdr_len > TCP_MIN_HEADER_LEN and not _default_accepts('TCP', segment[:hdr_len]))): + return None + src, dst = _addresses(layer) + flags = segment[13] + return _Segment( + src=src, + dst=dst, + srcport=int.from_bytes(segment[0:2], 'big'), + dstport=int.from_bytes(segment[2:4], 'big'), + seq=int.from_bytes(segment[4:8], 'big'), + ack=int.from_bytes(segment[8:12], 'big'), + rst=bool(flags & 0x04), + syn=bool(flags & 0x02), + fin=bool(flags & 0x01), + header=segment[:hdr_len], + payload=segment[hdr_len:], + ) - ip = _network(packet) - if ip is None: - return None, None - root = packet # the packet ``ip`` is a layer of - while True: - if isinstance(ip, dpkt.ip.IP): - if not _ipv4_accepted(ip): - return None, None - if ip.p in IPV4_EXTENSION_HEADERS: - upper = _past_ipv4_extension_headers(root, ip) - if upper is not None and upper[0] == Enum_TransType.TCP: - return ip, _decode_tcp(upper[1]) - if upper is None or upper[0] not in IP_TUNNELS or (inner := _decode_ip(*upper)) is None: - return ip, None - root = ip = inner - continue - if getattr(ip, 'p', None) not in IP_TUNNELS or not isinstance(ip.data, (dpkt.ip.IP, dpkt.ip6.IP6)): - break - ip = ip.data - tcp = getattr(ip, 'tcp', None) # type: Optional[TCP] - if tcp is None and type(getattr(ip, 'data', None)).__name__ == 'TCP': - tcp = cast('TCP', ip.data) - if tcp is not None and root is not packet: - attach_buffer(tcp, _layer2bytes(root, tcp)[0]) - return ip, tcp +def _addresses(layer: '_IPLayer') -> 'tuple[IPv4Address | IPv6Address, IPv4Address | IPv6Address]': + """Read the source and destination addresses of an IP layer. + + Args: + layer: IPv4 or IPv6 layer. + + Returns: + The source address, then the destination. + + """ + octets = layer.octets + if layer.version == 4: + return ipaddress.ip_address(octets[12:16]), ipaddress.ip_address(octets[16:20]) + return ipaddress.ip_address(octets[8:24]), ipaddress.ip_address(octets[24:40]) def ipv6_hdr_len(ipv6: 'IP6') -> 'int': @@ -868,14 +1117,16 @@ def packet2dict(packet: 'Packet', timestamp: 'float', *, is mapped as a layer of its own only where it is one, i.e. has the header fields a :class:`dpkt.dpkt.Packet` has; any other is leaf data. `DPKT`_ leaves some undecoded payloads as a :obj:`str` rather than as - :obj:`bytes`, e.g. ``''`` for VRRP's. + :obj:`bytes`, e.g. ``''`` for VRRP's. A frame the engine keeps as raw + data, as it keeps one `DPKT`_ cannot decode, has no header fields, so + maps to an empty :obj:`dict`. .. _DPKT: https://dpkt.readthedocs.io """ def wrapper(packet: 'Packet') -> 'dict[str, Any]': dict_ = {} # type: dict[str, Any] - for field in packet.__hdr_fields__: + for field in getattr(packet, '__hdr_fields__', ()): dict_[field] = getattr(packet, field, None) payload = packet.data if hasattr(payload, '__hdr_fields__'): @@ -889,143 +1140,6 @@ def wrapper(packet: 'Packet') -> 'dict[str, Any]': } -def _first_ipv4(packet: 'Packet') -> 'tuple[Optional[IP], Optional[int]]': - """Fetch the first IPv4 layer of a DPKT packet, as the default engine dissects it. - - Args: - packet: DPKT packet, i.e. the outermost layer. - - Returns: - The network layer (see :func:`_network`) if it is IPv4, else the first - IPv4 packet its IPv6 packet carries (:rfc:`2473`) -- directly, behind - extension headers, or inside more IPv6. Beside it, :data:`None` for the - former, and for the latter how many of its octets, from its first on, - lie inside the Payload Length of every IPv6 packet carrying it. Both are - :data:`None` if there is no such IPv4 packet, or the default engine - dissects nothing past an IPv6 packet on the way: a later fragment - (:rfc:`8200#section-4.5`), one with more than - :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` - extension headers (:issue:`1604`), or one whose Payload Length is zero - with no Jumbo Payload option to stand in for it (:rfc:`2675`). - - Note: - The default engine takes ``frame['IPv4']``, the first IPv4 layer however - deep (:func:`pcapkit.toolkit.pcap.ipv4_reassembly`); TCP is keyed by the - innermost instead (see :func:`_tcp_carrier`). It hands that layer only - the octets inside each Payload Length above it, so they end there, short - of any trailer (:issue:`1615`). `DPKT`_ decodes the IPv4 packet in all - three cases above -- a later fragment unless its Fragment header comes - first, any number of extension headers, and a zero Payload Length as the - rest of the frame -- so each is checked here. - - .. _DPKT: https://dpkt.readthedocs.io - - """ - import dpkt # isort:skip - - from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT # isort:skip - from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length # isort:skip - - ip = _network(packet) - extent = None # type: Optional[int] # octets of ``ip`` the packets above hold - while isinstance(ip, dpkt.ip6.IP6): - ext_hdrs = _ipv6_ext_hdrs(ip) - if len(ext_hdrs) > EXTENSION_HEADER_LIMIT or any( - isinstance(ext_hdr, dpkt.ip6.IP6FragmentHeader) and ext_hdr.frag_off for ext_hdr in ext_hdrs - ): - return None, None - if getattr(ip, 'p', None) not in IP_TUNNELS or not isinstance(ip.data, (dpkt.ip.IP, dpkt.ip6.IP6)): - return None, None - - plen = ip.plen - if not plen: - plen = jumbo_payload_length(ip.nxt, _layer2bytes(packet, ip)[0][ip.__hdr_len__:]) - if plen is None: - return None, None - end = ip.__hdr_len__ + plen - if extent is not None: - end = min(end, extent) - extent = max(end - _wire_hdr_len(ip, ip.data), 0) - ip = ip.data - - if not isinstance(ip, dpkt.ip.IP): - return None, None - return ip, extent - - -def _first_ipv6(packet: 'Packet') -> 'tuple[Optional[IP6], Optional[tuple[bytes, bool]], int]': - """Fetch the first IPv6 layer of a DPKT packet, as the default engine dissects it. - - Args: - packet: DPKT packet, i.e. the outermost layer. - - Returns: - The network layer (see :func:`_network`) if it is IPv6, else the first - IPv6 packet its IPv4 packet carries (:rfc:`4213`) -- directly, behind - the headers the default engine reads past (see - :func:`~pcapkit.toolkit.pypcapfile._past_extension_headers`), or inside - more IPv4 (:rfc:`2003`). Beside it, :data:`None` for the former, whose - octets are those :func:`_layer2bytes` slices, and for the latter its - octets from its first on, as far as the Total Length of every IPv4 - packet carrying it goes, and whether they are the captured ones; then - its place in the frame (see :func:`_network_depth`), one more for each - IPv4 packet and each of those headers before it. Both are :data:`None`, - and the place ``0``, if there is no such packet, or the default engine - dissects nothing past an IPv4 packet on the way: one whose header it - rejects (see :func:`_ipv4_accepted`), a later fragment - (:rfc:`791#section-3.2`), or one carrying any other protocol, such as - the ICMP error that quotes an IPv6 packet. So too if the IPv6 packet is - past :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`, which the - default engine keeps raw (:issue:`1610`). - - Note: - The default engine takes ``frame['IPv6']``, the first IPv6 layer however - deep (:func:`pcapkit.toolkit.pcap.ipv6_reassembly`), and an IPv4 packet - hands its payload the octets its Total Length leaves -- a Total Length - of ``0``, which TCP segmentation offload leaves, the rest of what - carries it -- so they are cut here as it cuts them (:issue:`1637`). - `DPKT`_ decodes the IPv6 packet an IPv4 packet carries directly, but - reads protocol 0 as IPv4 and keeps the IPv6 extension headers as octets, - so one behind an AH or extension header is decoded from the octets - instead (see :func:`_decode_ip`), as in :func:`_tcp_carrier`. - - .. _DPKT: https://dpkt.readthedocs.io - - """ - import dpkt # isort:skip - - from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT # isort:skip - - ip = _network(packet) - if ip is None: - return None, None, 0 - octets = None # type: Optional[tuple[bytes, bool]] # of ``ip``, once sliced - depth = _network_depth(packet) # of ``ip`` - while isinstance(ip, dpkt.ip.IP) and depth <= FRAME_LAYER_LIMIT: - if ip.p not in IP_TUNNELS and ip.p not in IPV4_EXTENSION_HEADERS: - return None, None, 0 - if not _ipv4_accepted(ip) or ip.offset: - return None, None, 0 - wire, captured = octets or _layer2bytes(packet, ip) - ihl = ip.hl * 4 - payload = wire[ihl:max(ip.len, ihl)] if ip.len else wire[ihl:] - if ip.p in IPV4_EXTENSION_HEADERS: - walked = _walk_extension_headers(ip.p, payload) - if walked is None or walked[0] not in IP_TUNNELS: - return None, None, 0 - proto, upper, hops = walked - if (inner := _decode_ip(proto, upper)) is None: - return None, None, 0 - ip, octets, depth = inner, (upper, captured), depth + 1 + hops - elif isinstance(ip.data, (dpkt.ip.IP, dpkt.ip6.IP6)): - ip, octets, depth = ip.data, (payload, captured), depth + 1 - else: - return None, None, 0 - if not hasattr(ip, 'extension_hdrs') or depth > FRAME_LAYER_LIMIT: - return None, None, 0 - return cast('IP6', ip), octets, depth - - def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, count: 'int' = -1) -> 'IP_Packet[IPv4Address] | None': """Make data for IPv4 reassembly. @@ -1046,11 +1160,9 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, Data for IPv4 reassembly. * If the ``packet`` can be used for IPv4 reassembly. A packet can be reassembled - if it contains IPv4 layer (:class:`dpkt.ip.IP`) -- the first of the frame, as - for the default engine, so one carried in IPv6 counts (see - :func:`_first_ipv4`) -- whose header the default engine accepts (see - :func:`_ipv4_accepted`), and the **DF** (:attr:`dpkt.ip.IP.df`) flag is - :data:`False`. + if it contains an IPv4 layer the default engine dissects -- the first of the + frame, as for the default engine, so one carried in IPv6 counts (see + :func:`_first_layer`) -- and its **DF** flag is :data:`False`. * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for IPv4 reassembly (:term:`reasm.ipv4.packet`) will be returned; otherwise, returns :data:`None`. @@ -1073,42 +1185,34 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.reassembly.ipv4.IPv4` """ - ipv4, extent = _first_ipv4(packet) - if ipv4 is not None and _ipv4_accepted(ipv4): - if ipv4.df: # dismiss not fragmented packet - return None - # internet header length, in octets -- ``IP.hl`` counts 32-bit words and - # covers any IP options, whereas ``IP.__hdr_len__`` is the fixed 20-octet - # struct size and would leave option octets at the head of the payload - ihl = ipv4.hl * 4 - wire, _ = _layer2bytes(packet, ipv4) - if extent is not None: - wire = wire[:extent] - # a Jumbo Payload Length may leave less than DPKT decoded from the - # rest of the frame, and the default engine then dissects no IPv4 - if len(wire) < ihl: - return None - - data = IP_Packet( - bufid=( - cast('IPv4Address', - ipaddress.ip_address(ipv4.src)), # source IP address - cast('IPv4Address', - ipaddress.ip_address(ipv4.dst)), # destination IP address - ipv4.id, # identification - Enum_TransType.get(ipv4.p), # payload protocol type - ), - num=count, # original packet range number - fo=ipv4.offset * 8, # fragment offset - ihl=ihl, # internet header length - mf=bool(ipv4.mf), # more fragment flag - tl=ipv4.len, # total length, header includes - header=wire[:ihl], # raw bytes type header - payload=bytearray(wire[ihl:]), # raw bytearray type payload - timestamp=float(timestamp), # capture timestamp - ) - return data - return None + ipv4 = _first_layer(packet, 4) + if ipv4 is None: + return None + octets = ipv4.octets + flags = int.from_bytes(octets[6:8], 'big') + if flags & 0x4000: # dismiss not fragmented packet + return None + src, dst = _addresses(ipv4) + # internet header length, in octets, options included + ihl = (octets[0] & 0x0F) * 4 + + data = IP_Packet( + bufid=( + cast('IPv4Address', src), # source IP address + cast('IPv4Address', dst), # destination IP address + int.from_bytes(octets[4:6], 'big'), # identification + Enum_TransType.get(octets[9]), # payload protocol type + ), + num=count, # original packet range number + fo=(flags & 0x1FFF) * 8, # fragment offset + ihl=ihl, # internet header length + mf=bool(flags & 0x2000), # more fragment flag + tl=int.from_bytes(octets[2:4], 'big'), # total length, header includes + header=octets[:ihl], # raw bytes type header + payload=bytearray(octets[ihl:]), # raw bytearray type payload + timestamp=float(timestamp), # capture timestamp + ) + return data def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, @@ -1126,14 +1230,10 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, Data for IPv6 reassembly. * If the ``packet`` can be used for IPv6 reassembly. A packet can be reassembled - if it contains IPv6 layer (:class:`dpkt.ip6.IP6`) -- the first of the frame, as - for the default engine, so one carried in IPv4 counts (see :func:`_first_ipv6`) - -- and IPv6 Fragment header (:rfc:`2460#section-4.5`, i.e., - :class:`dpkt.ip6.IP6FragmentHeader`) inside its Payload Length, and among its first - :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` extension - headers, the most the default engine walks (:issue:`1604`), and inside the first - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the frame, past - which it dissects nothing (:issue:`1610`). + if it contains an IPv6 layer the default engine dissects -- the first of the + frame, as for the default engine, so one carried in IPv4 counts (see + :func:`_first_layer`) -- and an IPv6 Fragment header (:rfc:`2460#section-4.5`) + among the extension headers the default engine walks (see :func:`_ipv6_chain`). * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for IPv6 reassembly (:term:`reasm.ipv6.packet`) will be returned; otherwise, returns :data:`None`. @@ -1146,103 +1246,54 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, :func:`attach_buffer` is what makes them the wire's again. The packet itself is left untouched either way. - The captured octets end at the Payload Length, as the default engine - cuts them, or at the Jumbo Payload Length (:rfc:`2675`) of a zero one. - A zero Payload Length with no Jumbo Payload option leaves the default - engine no extension header to read, and so no input; `DPKT`_ reads the - rest of the frame instead. + The header is the octets before the first Fragment header, and the + payload those after it, as far as the Payload Length goes, or the Jumbo + Payload Length (:rfc:`2675`) of a zero one, as the default engine cuts + them. See Also: :class:`pcapkit.foundation.reassembly.ipv6.IPv6` - .. _DPKT: https://dpkt.readthedocs.io - """ - from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT # isort:skip - from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT # isort:skip - from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length # isort:skip - - ipv6, octets, depth = _first_ipv6(packet) - if ipv6 is not None: - ipv6_frag = ipv6.extension_hdrs.get(44) # type: Optional[IP6FragmentHeader] - if ipv6_frag is None: # dismiss not fragmented packet - return None - # the default engine walks no more extension headers than either bound - # leaves, each header one layer of the frame past the IPv6 header's - bound = min(EXTENSION_HEADER_LIMIT, FRAME_LAYER_LIMIT - depth) - if not any(ext_hdr is ipv6_frag for ext_hdr in _ipv6_ext_hdrs(ipv6)[:bound]): - return None - hdr_len = ipv6_hdr_len(ipv6) - wire, captured = octets or _layer2bytes(packet, ipv6) - if captured: - plen = ipv6.plen or jumbo_payload_length(ipv6.nxt, wire[ipv6.__hdr_len__:]) - if not plen: - return None - wire = wire[:ipv6.__hdr_len__ + plen] - if len(wire) < hdr_len + ipv6_frag.__hdr_len__: # Fragment header cut short - return None - # payload following the IPv6 Fragment header - payload = wire[hdr_len + ipv6_frag.__hdr_len__:] - - data = IP_Packet( - bufid=( - cast('IPv6Address', - ipaddress.ip_address(ipv6.src)), # source IP address - cast('IPv6Address', - ipaddress.ip_address(ipv6.dst)), # destination IP address - # NOTE: The reassembly key is the Fragment header's Identification - # (:rfc:`8200#section-4.5`), not the IPv6 header's Flow Label. The - # label is optional and routinely zero, so keying on it collapses - # every datagram between one address pair into a single buffer and - # interleaves their fragments; it would also disagree with - # ``bufid[2]`` in every other engine, which feeds - # :attr:`pcapkit.foundation.reassembly.data.ip.DatagramID.id`. - ipv6_frag.id, # identification - Enum_TransType.get(ipv6_frag.nxt), # next header field in IPv6 Fragment Header - ), - num=count, # original packet range number - # NOTE: ``IP6FragmentHeader.frag_off`` is a ``__bit_fields__`` property - # over the 13-bit on-wire Fragment Offset, i.e. already shifted out of - # the flags word, so it counts 8-octet units (:rfc:`8200#section-4.5`). - # The reassembly machinery indexes the datagram buffer with ``fo``, so - # the units have to become octets here, exactly as - # :func:`pcapkit.toolkit.scapy.ipv6_reassembly` does. - fo=ipv6_frag.frag_off * 8, # fragment offset - ihl=hdr_len, # header length, only headers before IPv6-Frag - mf=bool(ipv6_frag.m_flag), # more fragment flag - tl=hdr_len + len(payload), # total length, header includes - header=wire[:hdr_len], # raw bytes type header before IPv6-Frag - payload=bytearray(payload), # raw bytearray type payload after IPv6-Frag - timestamp=float(timestamp), # capture timestamp - ) - return data - return None - - -def _tcp_flags(tcp: 'TCP') -> 'tuple[bool, bool, bool]': - """Read the RST, SYN and FIN flags of a DPKT TCP segment. - - Args: - tcp: DPKT TCP packet. - - Returns: - Whether RST, SYN and FIN are set, in that order. - - Note: - Each flag is masked out of :attr:`dpkt.tcp.TCP.flags` by its ``TH_*`` - value, so it is read from the flags octet alone, as the default engine - reads it. `DPKT`_ 1.9.8 makes that field nine bits wide, its top bit - the NS bit (:rfc:`3540`): read by its position among the field's binary - digits, every flag was one bit low with NS set, a SYN reading as a FIN - (:issue:`1618`). - - .. _DPKT: https://dpkt.readthedocs.io - - """ - from dpkt.tcp import TH_FIN, TH_RST, TH_SYN # isort:skip - - flags = tcp.flags - return bool(flags & TH_RST), bool(flags & TH_SYN), bool(flags & TH_FIN) + ipv6 = _first_layer(packet, 6) + if ipv6 is None: + return None + octets, fragment, _ = _ipv6_chain(ipv6) + if fragment is None: # dismiss not fragmented packet + return None + src, dst = _addresses(ipv6) + word = int.from_bytes(octets[fragment + 2:fragment + 4], 'big') + # payload following the IPv6 Fragment header + payload = octets[fragment + 8:] + + data = IP_Packet( + bufid=( + cast('IPv6Address', src), # source IP address + cast('IPv6Address', dst), # destination IP address + # NOTE: The reassembly key is the Fragment header's Identification + # (:rfc:`8200#section-4.5`), not the IPv6 header's Flow Label. The + # label is optional and routinely zero, so keying on it collapses + # every datagram between one address pair into a single buffer and + # interleaves their fragments; it would also disagree with + # ``bufid[2]`` in every other engine, which feeds + # :attr:`pcapkit.foundation.reassembly.data.ip.DatagramID.id`. + int.from_bytes(octets[fragment + 4:fragment + 8], 'big'), # identification + Enum_TransType.get(octets[fragment]), # next header field in IPv6 Fragment Header + ), + num=count, # original packet range number + # NOTE: The Fragment Offset is the top 13 bits of its word, in 8-octet + # units (:rfc:`8200#section-4.5`). The reassembly machinery indexes the + # datagram buffer with ``fo``, so the units have to become octets here, + # exactly as :func:`pcapkit.toolkit.scapy.ipv6_reassembly` does. + fo=(word >> 3) * 8, # fragment offset + ihl=fragment, # header length, only headers before IPv6-Frag + mf=bool(word & 0x1), # more fragment flag + tl=fragment + len(payload), # total length, header includes + header=octets[:fragment], # raw bytes type header before IPv6-Frag + payload=bytearray(payload), # raw bytearray type payload after IPv6-Frag + timestamp=float(timestamp), # capture timestamp + ) + return data def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, @@ -1260,9 +1311,8 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, Data for TCP reassembly. * If the ``packet`` can be used for TCP reassembly. A packet can be reassembled - if it contains TCP layer (:class:`dpkt.tcp.TCP`), keyed by the IP packet that - carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_tcp_carrier`) - -- and behind no IPv4 header the default engine rejects. + if it contains a TCP segment the default engine reads, keyed by the IP packet + that carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_tcp_carrier`). * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP reassembly (:term:`reasm.tcp.packet`) will be returned; otherwise, returns :data:`None`. @@ -1279,29 +1329,27 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.reassembly.tcp.TCP` """ - ip, tcp = _tcp_carrier(packet) - if ip is not None and tcp is not None: - rst, syn, fin = _tcp_flags(tcp) - wire, _ = _layer2bytes(packet, tcp) - raw_len = len(tcp.data) # payload length, header excludes + segment = _tcp_carrier(packet) + if segment is not None: + raw_len = len(segment.payload) # payload length, header excludes data = TCP_Packet( bufid=( - ipaddress.ip_address(ip.src), # source IP address - tcp.sport, # source port - ipaddress.ip_address(ip.dst), # destination IP address - tcp.dport, # destination port + segment.src, # source IP address + segment.srcport, # source port + segment.dst, # destination IP address + segment.dstport, # destination port ), num=count, # original packet range number - ack=tcp.ack, # acknowledgement - dsn=tcp.seq, # data sequence number - rst=rst, # reset connection flag - syn=syn, # synchronise flag - fin=fin, # finish flag - header=wire[:tcp.off * 4], # raw bytes type header - payload=bytearray(bytes(tcp.data)), # raw bytearray type payload - first=tcp.seq, # first sequence number of payload - last=tcp.seq + raw_len - 1, # last sequence number of payload + ack=segment.ack, # acknowledgement + dsn=segment.seq, # data sequence number + rst=segment.rst, # reset connection flag + syn=segment.syn, # synchronise flag + fin=segment.fin, # finish flag + header=segment.header, # raw bytes type header + payload=bytearray(segment.payload), # raw bytearray type payload + first=segment.seq, # first sequence number of payload + last=segment.seq + raw_len - 1, # last sequence number of payload len=raw_len, # payload length, header excludes timestamp=float(timestamp), # capture timestamp ) @@ -1326,9 +1374,8 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, Data for TCP reassembly. * If the ``packet`` can be used for TCP flow tracing. A packet can be reassembled - if it contains TCP layer (:class:`dpkt.tcp.TCP`), keyed by the IP packet that - carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_tcp_carrier`) - -- and behind no IPv4 header the default engine rejects. + if it contains a TCP segment the default engine reads, keyed by the IP packet + that carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_tcp_carrier`). * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`. @@ -1345,27 +1392,24 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, :class:`pcapkit.foundation.traceflow.tcp.TCP` """ - ip, tcp = _tcp_carrier(packet) - if ip is not None and tcp is not None: - rst, syn, fin = _tcp_flags(tcp) - wire, _ = _layer2bytes(packet, tcp) - + segment = _tcp_carrier(packet) + if segment is not None: data = TF_TCP_Packet( # type: ignore[type-var] protocol=data_link, # data link type from global header index=count, # frame number frame=packet2frame(packet, timestamp, data_link=data_link), # extracted packet - syn=syn, # TCP synchronise (SYN) flag - fin=fin, # TCP finish (FIN) flag - rst=rst, # TCP reset (RST) flag - src=ipaddress.ip_address(ip.src), # source IP - dst=ipaddress.ip_address(ip.dst), # destination IP - srcport=tcp.sport, # TCP source port - dstport=tcp.dport, # TCP destination port + syn=segment.syn, # TCP synchronise (SYN) flag + fin=segment.fin, # TCP finish (FIN) flag + rst=segment.rst, # TCP reset (RST) flag + src=segment.src, # source IP + dst=segment.dst, # destination IP + srcport=segment.srcport, # TCP source port + dstport=segment.dstport, # TCP destination port timestamp=float(timestamp), # timestamp - seq=tcp.seq, # TCP sequence number - ack=tcp.ack, # TCP acknowledgement number - header=wire[:tcp.off * 4], # raw bytes type header - payload=bytearray(bytes(tcp.data)), # raw bytearray type payload + seq=segment.seq, # TCP sequence number + ack=segment.ack, # TCP acknowledgement number + header=segment.header, # raw bytes type header + payload=bytearray(segment.payload), # raw bytearray type payload ) return data return None diff --git a/tests/toolkit/test_dpkt_ip_walk_1609_1626_1651_1654_unit.py b/tests/toolkit/test_dpkt_ip_walk_1609_1626_1651_1654_unit.py new file mode 100644 index 0000000000..a516eed7a7 --- /dev/null +++ b/tests/toolkit/test_dpkt_ip_walk_1609_1626_1651_1654_unit.py @@ -0,0 +1,365 @@ +# -*- coding: utf-8 -*- +"""dpkt reads a frame's IP layers off its octets, as the default engine does. C.f. #1609, #1626, #1651, #1654. + +The dpkt adapters walked what dpkt decoded, which is not what the default engine +dissects: + +* #1651: dpkt runs an IPv6 payload to the end of the record when the Payload + Length is zero, and past a jumbogram's Jumbo Payload Length, so TCP and IPv6 + fragments were read where the default engine reads none, or longer. +* #1654: dpkt reads at most two VLAN tags a pass and two passes, so behind a + deeper stack it decoded nothing, and the adapters handed over nothing. +* #1609: the default engine walks 32 extension headers in a row and dissects 64 + layers of a frame; the walk for TCP and IPv4 reassembly bounded neither. +* #1626: dpkt decodes none of Mobility, HIP and Shim6, cannot read a later + fragment whose Next Header is an extension header, and raises at a Fragment + header first in the chain with another header behind it, which keeps the whole + frame raw. + +The adapters now read every IP layer, extension header and TCP segment off the +captured octets by the default engine's rules, and a frame the engine keeps raw +by the link type it attaches. Every input to TCP reassembly, flow tracing and IP +reassembly is compared with the default engine's, field for field. A frame kept +raw is written out, and traced, rather than ending the extraction. + +Every capture is built here, so the module is in the unit tier. + +""" +from __future__ import annotations + +import importlib.util +import itertools +import os +import tempfile +import unittest +import warnings +from typing import TYPE_CHECKING + +from tests._support import close_extractor, reimport_once_per_class +from tests.foundation import _roundtrip as wire +from tests.toolkit.test_dpkt_6in4_reassembly_and_mpls_isl_1637_1638_unit import (TRAILER, nested, + numbered, read, + tagged) +from tests.toolkit.test_dpkt_ipv4_reassembly_input_1615_1616_unit import ipv6, jumbo +from tests.toolkit.test_tcp_carrier_walk_1606_1617_unit import ah, eth, options, tcp, v4, v6 + +if TYPE_CHECKING: + from typing import Any, Callable + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) +HAS_DPKT = importlib.util.find_spec('dpkt') is not None + +#: What each case's counts count, in order. +ASPECTS = ('TCP', 'trace', 'IPv4', 'IPv6') + +#: TCP ports and fragment Identifications, one per use. +_NUMBERS = itertools.count(1) + + +def number() -> int: + """A port offset or Identification no other frame uses.""" + return next(_NUMBERS) + + +def segment() -> bytes: + """A TCP segment carrying four octets, on a port pair of its own.""" + return tcp(number()) + + +def frag6(nxt: int, ident: int, *, offset: int = 0, mf: bool = True) -> bytes: + """An IPv6 Fragment header; ``offset`` is in octets.""" + return bytes([nxt, 0]) + (offset | mf).to_bytes(2, 'big') + ident.to_bytes(4, 'big') + + +def v4_fragments(carry: 'Callable[[bytes], bytes]') -> 'list[bytes]': + """The two fragments of a 24-octet IPv4 datagram, each carried by ``carry``.""" + ident = number() + return [carry(v4(b'A' * 16, 253, mf=True, ident=ident)), carry(v4(b'B' * 8, 253, offset=16, ident=ident))] + + +def v6_fragments(carry: 'Callable[[bytes], bytes]') -> 'list[bytes]': + """The two fragments of a 24-octet IPv6 datagram, each carried by ``carry``.""" + ident = number() + return [carry(v6(frag6(253, ident) + b'A' * 16, 44)), + carry(v6(frag6(253, ident, offset=16, mf=False) + b'B' * 8, 44))] + + +def nested6(packet: bytes, count: int, nxt: int) -> bytes: + """``packet`` carried as ``nxt`` in the innermost of ``count`` nested IPv6 headers.""" + packet = v6(packet, nxt) + for _ in range(count - 1): + packet = v6(packet, 41) + return packet + + +def pairs(segment: bytes, count: int, *, version: int) -> bytes: + """``segment`` behind ``count`` (IPv4, AH) or (IPv6, Hop-by-Hop) pairs, as ``version`` says.""" + if version == 4: + packet = v4(ah(6) + segment, 51) + for _ in range(count - 1): + packet = v4(ah(4) + packet, 51) + return packet + packet = v6(options(6) + segment, 0) + for _ in range(count - 1): + packet = v6(options(41) + packet, 0) + return packet + + +def mobility(nxt: int) -> bytes: + """A Mobility header (135) of 8 octets, a Binding Refresh Request.""" + return bytes([nxt, 0, 0, 0, 0, 0, 0, 0]) + + +def hip(nxt: int) -> bytes: + """A HIP header (139) of 40 octets, an I1 packet.""" + return bytes([nxt, 4, 1, 0x21, 0, 0, 0, 0]) + b'\x11' * 32 + + +def shim6(nxt: int) -> bytes: + """A Shim6 header (140) of 8 octets.""" + return bytes([nxt, 0, 0, 0, 0, 0, 0, 0]) + + +def payload_length() -> 'dict[str, tuple[tuple[int, ...], list[bytes]]]': + """#1651: IPv6 packets whose Payload Length is zero, each with the default engine's input counts.""" + def jumbogram(short: int = 0) -> bytes: + return ipv6(jumbo(6, 8 + 24 - short) + segment(), 0, plen=0) + + return { + 'IPv6, TCP (control)': ((1, 1, 0, 0), [eth(v6(segment(), 6))]), + 'Payload Length 0, TCP': ((0, 0, 0, 0), [eth(ipv6(segment(), 6, plen=0))]), + 'Payload Length 0, TCP, trailer': ((0, 0, 0, 0), [eth(ipv6(segment(), 6, plen=0)) + TRAILER]), + '6in4, Payload Length 0, TCP': ((0, 0, 1, 0), [eth(v4(ipv6(segment(), 6, plen=0), 41))]), + 'Payload Length 0, Fragment header': ((0, 0, 0, 0), [eth(ipv6(frag6(253, number()) + b'A' * 16, 44, + plen=0))]), + 'Payload Length 0, Fragment header, trailer': ((0, 0, 0, 0), [eth(ipv6(frag6(253, number()) + b'A' * 16, 44, + plen=0)) + TRAILER]), + 'Payload Length 0, Hop-by-Hop with no Jumbo Payload option, TCP': ( + (0, 0, 0, 0), [eth(ipv6(options(6) + segment(), 0, plen=0))]), + 'Payload Length 0, 4in6, TCP': ((0, 0, 0, 0), [eth(ipv6(v4(segment(), 6), 4, plen=0))]), + '6in6, inner Payload Length 0, TCP': ((0, 0, 0, 0), [eth(v6(ipv6(segment(), 6, plen=0), 41))]), + 'jumbogram, TCP (control)': ((1, 1, 0, 0), [eth(jumbogram())]), + 'jumbogram, TCP, trailer': ((1, 1, 0, 0), [eth(jumbogram()) + TRAILER]), + 'jumbogram, Jumbo Payload Length 2 short': ((1, 1, 0, 0), [eth(jumbogram(2))]), + 'jumbogram, Jumbo Payload Length inside the TCP header': ((0, 0, 0, 0), [eth(jumbogram(12))]), + '6in4 jumbogram, Jumbo Payload Length 2 short': ((1, 1, 1, 0), [eth(v4(jumbogram(2), 41))]), + } + + +#: VLAN tag stacks dpkt does not finish decoding in two passes, by name. +STACKS = { + 'three 802.1ad': [0x88A8] * 3, + 'five 802.1Q': [0x8100] * 5, + '802.1ad, four 802.1Q': [0x88A8] + [0x8100] * 4, + '802.1Q, two 802.1ad': [0x8100, 0x88A8, 0x88A8], + 'eight 802.1Q': [0x8100] * 8, +} + + +def vlan_stacks() -> 'dict[str, tuple[tuple[int, ...], list[bytes]]]': + """#1654: IP behind deep VLAN tag stacks, each with the default engine's input counts.""" + cases = {} # type: dict[str, tuple[tuple[int, ...], list[bytes]]] + for name, tpids in STACKS.items(): + def carry(packet: bytes, tpids: 'list[int]' = tpids) -> bytes: + return tagged(tpids, packet) + cases[f'{name}, IPv4, TCP'] = ((1, 1, 1, 0), [carry(v4(segment(), 6))]) + cases[f'{name}, IPv6, TCP'] = ((1, 1, 0, 0), [carry(v6(segment(), 6))]) + cases[f'{name}, IPv4 fragments'] = ((0, 0, 2, 0), v4_fragments(carry)) + cases[f'{name}, IPv6 fragments'] = ((0, 0, 0, 2), v6_fragments(carry)) + cases[f'{name}, 6in4 fragments'] = ((0, 0, 2, 2), v6_fragments(lambda p, c=carry: c(v4(p, 41)))) + cases['three 802.1ad, IPv4, TCP, trailer'] = ((1, 1, 1, 0), [tagged(STACKS['three 802.1ad'], + v4(segment(), 6)) + TRAILER]) + # where dpkt finishes in two passes, and where a legacy tag deep in the stack is left raw + cases['two 802.1ad, IPv4, TCP (control)'] = ((1, 1, 1, 0), [tagged([0x88A8] * 2, v4(segment(), 6))]) + cases['three 802.1ad, legacy QinQ 0x9100, IPv4, TCP'] = ((0, 0, 0, 0), [tagged([0x88A8] * 3 + [0x9100], + v4(segment(), 6))]) + return cases + + +def layer_limit() -> 'dict[str, tuple[tuple[int, ...], list[bytes]]]': + """#1609: TCP and IPv4 fragments deep in a frame, each with the default engine's input counts. + + The default engine walks 32 extension headers in a row (#1604) and dissects + 64 layers of a frame (#1610): Ethernet, then each VLAN tag, IP header, + header in IPv4 it reads past and IPv6 extension header. + + """ + cases = { + 'IPv6, 32 Hop-by-Hop headers, TCP (control)': ((1, 1, 0, 0), [eth(v6(options(0) * 31 + options(6) + + segment(), 0))]), + 'IPv6, 33 Hop-by-Hop headers, TCP': ((0, 0, 0, 0), [eth(v6(options(0) * 32 + options(6) + segment(), 0))]), + 'IPv6, 1000 Hop-by-Hop headers, TCP': ((0, 0, 0, 0), [eth(v6(options(0) * 999 + options(6) + + segment(), 0))]), + 'IPv6, 32 Destination Options headers, 4in6, TCP (control)': ( + (1, 1, 1, 0), [eth(v6(options(60) * 31 + options(4) + v4(segment(), 6), 60))]), + 'IPv6, 33 Destination Options headers, 4in6, TCP': ( + (0, 0, 0, 0), [eth(v6(options(60) * 32 + options(4) + v4(segment(), 6), 60))]), + 'TCP in 62 IPv4 (control)': ((1, 1, 1, 0), [eth(nested(segment(), 62, proto=6))]), + 'TCP in 63 IPv4': ((0, 0, 1, 0), [eth(nested(segment(), 63, proto=6))]), + 'TCP in 62 IPv6 (control)': ((1, 1, 0, 0), [eth(nested6(segment(), 62, 6))]), + 'TCP in 63 IPv6': ((0, 0, 0, 0), [eth(nested6(segment(), 63, 6))]), + 'TCP behind 31 (IPv4, AH) pairs (control)': ((1, 1, 1, 0), [eth(pairs(segment(), 31, version=4))]), + 'TCP behind 32 (IPv4, AH) pairs': ((0, 0, 1, 0), [eth(pairs(segment(), 32, version=4))]), + 'TCP behind 31 (IPv6, Hop-by-Hop) pairs (control)': ((1, 1, 0, 0), [eth(pairs(segment(), 31, version=6))]), + 'TCP behind 32 (IPv6, Hop-by-Hop) pairs': ((0, 0, 0, 0), [eth(pairs(segment(), 32, version=6))]), + 'IPv4 fragments in 62 IPv6 (control)': ((0, 0, 2, 0), v4_fragments(lambda p: eth(nested6(p, 62, 4)))), + 'IPv4 fragments in 63 IPv6': ((0, 0, 0, 0), v4_fragments(lambda p: eth(nested6(p, 63, 4)))), + 'TCP behind 61 802.1Q tags (control)': ((1, 1, 1, 0), [tagged([0x8100] * 61, v4(segment(), 6))]), + 'TCP behind 62 802.1Q tags': ((0, 0, 1, 0), [tagged([0x8100] * 62, v4(segment(), 6))]), + 'IPv4 fragments behind 62 802.1Q tags (control)': ((0, 0, 2, 0), + v4_fragments(lambda p: tagged([0x8100] * 62, p))), + 'IPv4 fragments behind 63 802.1Q tags': ((0, 0, 0, 0), v4_fragments(lambda p: tagged([0x8100] * 63, p))), + 'IPv6 fragments behind 61 802.1Q tags (control)': ((0, 0, 0, 2), + v6_fragments(lambda p: tagged([0x8100] * 61, p))), + 'IPv6 fragments behind 62 802.1Q tags': ((0, 0, 0, 0), v6_fragments(lambda p: tagged([0x8100] * 62, p))), + } + return cases + + +def raw_layer_limit() -> 'dict[str, tuple[tuple[int, ...], list[bytes]]]': + """#1609 on a raw-IP link type, whose first layer is the outermost IP header.""" + return { + 'TCP in 63 IPv4 (control)': ((1, 1, 1, 0), [nested(segment(), 63, proto=6)]), + 'TCP in 64 IPv4': ((0, 0, 1, 0), [nested(segment(), 64, proto=6)]), + } + + +def chains() -> 'dict[str, tuple[tuple[int, ...], list[bytes]]]': + """#1626: TCP and IP fragments behind IPv6 extension header chains, each with the default engine's input counts.""" + chain = { + 'Fragment (offset 0, M=0), then AH': (lambda nxt: frag6(51, number(), mf=False) + ah(nxt), 44, 1), + 'Fragment (offset 0, M=0), then Destination Options': ( + lambda nxt: frag6(60, number(), mf=False) + options(nxt), 44, 1), + 'AH': (ah, 51, 0), + 'Mobility (135)': (mobility, 135, 0), + 'HIP (139)': (hip, 139, 0), + 'Shim6 (140)': (shim6, 140, 0), + 'Destination Options (control)': (options, 60, 0), + } + cases = {} # type: dict[str, tuple[tuple[int, ...], list[bytes]]] + for name, (header, first, fragmented) in chain.items(): + cases[f'{name}, TCP'] = ((1, 1, 0, fragmented), [eth(v6(header(6) + segment(), first))]) + cases[f'{name}, TCP, trailer'] = ((1, 1, 0, fragmented), [eth(v6(header(6) + segment(), first)) + TRAILER]) + cases[f'6in4, {name}, TCP'] = ((1, 1, 1, fragmented), [eth(v4(v6(header(6) + segment(), first), 41))]) + cases[f'{name}, IPv4 fragments'] = ((0, 0, 2, 2 * fragmented), + v4_fragments(lambda p, h=header, f=first: eth(v6(h(4) + p, f)))) + # an IPv6 reassembly input whose Fragment header names another extension header + ident = number() + cases.update({ + 'Fragment (offset 0, M=1), then 253 (control)': ((0, 0, 0, 1), [eth(v6(frag6(253, number()) + + b'A' * 16, 44))]), + 'Fragment (offset 0, M=1), then Destination Options': ( + (0, 0, 0, 1), [eth(v6(frag6(60, number()) + options(253) + b'A' * 8, 44))]), + 'Fragment (offset 0, M=0), then Destination Options': ( + (0, 0, 0, 1), [eth(v6(frag6(60, number(), mf=False) + options(253) + b'A' * 8, 44))]), + 'Fragment (offset 0, M=1), then AH': ((0, 0, 0, 1), [eth(v6(frag6(51, number()) + ah(253) + b'A' * 8, 44))]), + 'Fragment (offset 8, M=0), next header 60': ( + (0, 0, 0, 1), [eth(v6(frag6(60, number(), offset=8, mf=False) + b'B' * 8, 44))]), + 'a datagram whose Fragmentable Part starts with Destination Options': ( + (0, 0, 0, 2), [eth(v6(frag6(60, ident) + options(253) + b'A' * 8, 44)), + eth(v6(frag6(60, ident, offset=16, mf=False) + b'B' * 8, 44))]), + 'two Fragment headers': ((0, 0, 0, 1), [eth(v6(frag6(44, number()) + frag6(253, number()) + b'A' * 16, 44))]), + 'a Hop-by-Hop header overrunning the packet, TCP': ( + (0, 0, 0, 0), [eth(v6(bytes([6, 5, 1, 4, 0, 0, 0, 0]) + segment(), 0))]), + '802.1Q, Fragment (offset 0, M=0), then AH, TCP': ( + (1, 1, 0, 1), [tagged([0x8100], v6(frag6(51, number(), mf=False) + ah(6) + segment(), 44))]), + }) + return cases + + +def raw_link_chains() -> 'dict[int, dict[str, tuple[tuple[int, ...], list[bytes]]]]': + """#1626 on the other link types, by link type, where dpkt's decoding raises and the frame is kept raw.""" + def first(nxt: int = 6) -> bytes: + return v6(frag6(51, number(), mf=False) + ah(nxt) + segment(), 44) + + def later() -> bytes: + return v6(frag6(60, number()) + options(253) + b'A' * 8, 44) + + return { + # and where the default engine dissects nothing past the link layer + 1: {'Ethernet, cut short of its header': ((0, 0, 0, 0), [eth(first())[:10]]), + 'Ethernet, 802.1Q tag cut short': ((0, 0, 0, 0), [wire.MAC_B + wire.MAC_A + b'\x81\x00\x00\x64'])}, + 101: {'raw IP, Fragment then AH, TCP': ((1, 1, 0, 1), [first()]), + 'raw IP, Fragment then Destination Options': ((0, 0, 0, 1), [later()]), + 'raw IP, version 5': ((0, 0, 0, 0), [bytes([0x55]) + v4(segment(), 6)[1:]]), + 'raw IP, no octets': ((0, 0, 0, 0), [b''])}, + 0: {'NULL, Fragment then AH, TCP': ((1, 1, 0, 1), [b'\x18\x00\x00\x00' + first()]), + 'NULL, Fragment then Destination Options': ((0, 0, 0, 1), [b'\x18\x00\x00\x00' + later()]), + 'NULL, cut short of its family': ((0, 0, 0, 0), [b'\x18\x00'])}, + 108: {'LOOP, Fragment then AH, TCP': ((1, 1, 0, 1), [b'\x00\x00\x00\x18' + first()])}, + 147: {'USER0, IPv4, TCP': ((0, 0, 0, 0), [v4(segment(), 6)])}, + 228: {'IPV4, 6in4, Fragment then AH, TCP': ((1, 1, 1, 1), [v4(first(), 41)])}, + 229: {'IPV6, Fragment then Destination Options': ((0, 0, 0, 1), [later()])}, + } + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +@unittest.skipUnless(HAS_DPKT, 'dpkt not installed') +class DPKTIPWalkTests(unittest.TestCase): + """dpkt's reassembly and trace inputs are the default engine's, field for field.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def agree(self, cases: 'dict[str, tuple[tuple[int, ...], list[bytes]]]', linktype: int = 1) -> None: + frames, numbers = numbered(cases) + base, dpkt = read(self, 'default', frames, linktype), read(self, 'dpkt', frames, linktype) + for label, (counts, _) in cases.items(): + with self.subTest(case=label, linktype=linktype): + self.assertEqual(tuple(sum(frame in base[aspect] for frame in numbers[label]) for aspect in ASPECTS), + counts, 'the default engine') + for aspect in ASPECTS: + self.assertEqual([dpkt[aspect].get(frame) for frame in numbers[label]], + [base[aspect].get(frame) for frame in numbers[label]], aspect) + for aspect in ('datagrams', 'flows'): + self.assertEqual(dpkt[aspect], base[aspect], aspect) + + def test_a_zero_payload_length_carries_no_payload_unless_a_jumbogram(self) -> None: + # #1651 + self.agree(payload_length()) + + def test_ip_behind_a_deep_vlan_tag_stack_is_read(self) -> None: + # #1654 + self.agree(vlan_stacks()) + + def test_tcp_and_ipv4_past_the_chain_and_frame_bounds_are_no_input(self) -> None: + # #1609 + self.agree(layer_limit()) + self.agree(raw_layer_limit(), linktype=101) + + def test_tcp_and_fragments_behind_every_extension_header_chain_are_read(self) -> None: + # #1626 + self.agree(chains()) + + def test_a_frame_dpkt_cannot_decode_is_read_by_its_link_type(self) -> None: + # #1626: dpkt raises AttributeError for a Fragment header first in the + # chain with another header behind it, so the engine keeps the frame raw + for linktype, cases in raw_link_chains().items(): + self.agree(cases, linktype=linktype) + + def test_a_frame_kept_raw_is_written_out(self) -> None: + # The engine writes every frame through packet2dict, which read the + # header fields a frame kept raw does not have, so AttributeError ended + # the extraction at the first frame dpkt could not decode + from pcapkit import extract + + frames = [eth(v6(frag6(51, number(), mf=False) + ah(6) + segment(), 44)), eth(v4(segment(), 6))] + with tempfile.TemporaryDirectory() as tmp, warnings.catch_warnings(): + warnings.simplefilter('ignore') + path = os.path.join(tmp, 'raw.pcap') + with open(path, 'wb') as file: + file.write(wire.pcap([(1, index, octets) for index, octets in enumerate(frames)])) + extractor = extract(fin=path, fout=os.path.join(tmp, 'out'), format='json', engine='dpkt', + store=True) + try: + if extractor._exnam != 'dpkt': # pylint: disable=protected-access + self.skipTest('dpkt did not run') + self.assertEqual(len(extractor.frame), 2) + self.assertEqual(type(extractor.frame[0]).__name__, 'RawPacket') + finally: + close_extractor(extractor) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/toolkit/test_dpkt_reassembly_rawbytes_unit.py b/tests/toolkit/test_dpkt_reassembly_rawbytes_unit.py index db18d9ddd8..3e1706de9b 100644 --- a/tests/toolkit/test_dpkt_reassembly_rawbytes_unit.py +++ b/tests/toolkit/test_dpkt_reassembly_rawbytes_unit.py @@ -12,10 +12,9 @@ ``len()``: :meth:`dpkt.ip6.IP6.__len__` sums ``len()`` of the extension headers rather than their wire lengths, which is 12 octets plus the rest of the frame for an Authentication header and 8 octets short for a Routing header whose Hdr Ext -Len is odd. :class:`~dpkt.ip6.IP6ExtensionHeader` records the wire length in -``length``, which is what :meth:`dpkt.ip6.IP6.unpack` walks the buffer by, so -that is what the offset is built from -- and the slice is then checked against -the layer's own header before it is handed back. +Len is odd. So the extension headers are walked off the octets, by the lengths +their own fields declare (#1626), and the network layer's slice is checked +against its own header before anything is read from it. Every frame below is written to a PCAP file and read back through the engine, so the tests exercise the path a capture actually takes rather than a packet with a @@ -357,10 +356,11 @@ def test_the_unbuffered_fallback_returns_dpkts_serialisation(self) -> None: self.assertEqual(bytes(data.header) + bytes(data.payload), IPV4_TCP_REPACKED) self.assertNotEqual(IPV4_TCP_REPACKED, IPV4_TCP) - # the TCP layer alone round-trips: dpkt.tcp.TCP.__bytes__ recomputes - # nothing, the pseudo-header checksum being dpkt.ip.IP.__bytes__'s doing + # the TCP segment is read out of the same serialisation of the network + # layer, so it carries the checksum dpkt.ip.IP.__bytes__ fills in with + # the IPv4 header's (#1626) tcp = toolkit.tcp_reassembly(dpkt.ethernet.Ethernet(ETH_IPV4 + IPV4_TCP), 0.0) - self.assertEqual(bytes(tcp.header), IPV4_TCP[20:40]) + self.assertEqual(bytes(tcp.header), IPV4_TCP_REPACKED[20:40]) def test_attaching_the_buffer_makes_the_octets_the_wires_again(self) -> None: import dpkt diff --git a/tests/toolkit/test_dpkt_unit.py b/tests/toolkit/test_dpkt_unit.py index 81bccb3356..6aa19334c4 100644 --- a/tests/toolkit/test_dpkt_unit.py +++ b/tests/toolkit/test_dpkt_unit.py @@ -76,42 +76,27 @@ def pack(self) -> bytes: return b'I' * 40 + b'H' * 8 + b'F' * 8 + b'PAYLOAD' -class TCP: - __hdr_fields__ = ('sport', 'dport', 'seq', 'ack', 'flags') - __hdr_len__ = 4 - #: Data Offset, in 32-bit words. ``off * 4`` has to agree with the header - #: portion of :meth:`pack`, the way it does on a real :class:`dpkt.tcp.TCP`. - off = 1 - sport = 2345 - dport = 443 - seq = 20 - ack = 15 - flags = 0b00010011 - data = b'ip6' - - def pack(self) -> bytes: - return b'HEADip6' - - -class FakeTCPIPv6: - __hdr_fields__ = ('src', 'dst') - src = ip_address('2001:db8::10').packed - dst = ip_address('2001:db8::20').packed - - def __init__(self) -> None: - self.data = TCP() +def _ipv6_frame(payload: bytes, nxt: int, *, src: str, dst: str, flow: int = 0): + """Build an Ethernet frame carrying IPv6, as the DPKT engine hands it over: its record octets attached.""" + import dpkt + from pcapkit.toolkit import dpkt as toolkit -class FakeDPKTPacket: - __hdr_fields__ = ('link',) - link = 1 + octets = (b'\xbb' * 6 + b'\xaa' * 6 + b'\x86\xdd' + + struct.pack('>IHBB16s16s', 0x60000000 | flow, len(payload), nxt, 64, + ip_address(src).packed, ip_address(dst).packed) + payload) + packet = dpkt.ethernet.Ethernet(octets) + toolkit.attach_buffer(packet, octets) + return packet - def __init__(self) -> None: - self.ip6 = FakeTCPIPv6() - self.data = self.ip6 - def pack(self) -> bytes: - return b'packet' +def _fake_fragment_frame(): + """A later IPv6 fragment, behind a Hop-by-Hop header, with the fields :class:`FakeFragment` pins.""" + frag = FakeFragment() + payload = (bytes([44, 0, 1, 4, 0, 0, 0, 0]) # Hop-by-Hop, one PadN + + struct.pack('>BBHI', frag.nxt, 0, frag.frag_off << 3 | frag.m_flag, frag.id) + + b'PAYLOAD') + return _ipv6_frame(payload, 0, src='2001:db8::1', dst='2001:db8::2', flow=FakeIPv6.flow) @unittest.skipUnless(HAS_RUNTIME and HAS_DPKT, 'runtime dependencies not installed') @@ -233,8 +218,7 @@ def test_every_adapter_hands_on_a_float_timestamp(self) -> None: results = { 'ipv4': toolkit.ipv4_reassembly(self._make_ipv4_tcp_packet(fragmented=True), stamp, count=1), - 'ipv6': toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=FakeIPv6(FakeFragment())), - stamp, count=1), + 'ipv6': toolkit.ipv6_reassembly(_fake_fragment_frame(), stamp, count=1), 'tcp': toolkit.tcp_reassembly(packet, stamp, count=1), 'trace': toolkit.tcp_traceflow(packet, stamp, data_link=LinkType.ETHERNET, count=1), } @@ -294,11 +278,13 @@ def fields(data): # type: ignore[no-untyped-def] (99, True, bytes(20))) self.assertIsNone(toolkit.ipv4_reassembly(ipv6, 1.0, count=1)) - def test_tcp_helpers_cover_ipv6_and_data_payload_fallback(self) -> None: + def test_tcp_helpers_cover_tcp_over_ipv6(self) -> None: from pcapkit.const.reg.linktype import LinkType from pcapkit.toolkit import dpkt as toolkit - packet = FakeDPKTPacket() + # SYN and FIN set, with ACK + segment = struct.pack('>HHIIBBHHH', 2345, 443, 20, 15, 5 << 4, 0b00010011, 1024, 0, 0) + b'ip6' + packet = _ipv6_frame(segment, 6, src='2001:db8::10', dst='2001:db8::20') tcp = toolkit.tcp_reassembly(packet, 60.5, count=12) self.assertIsNotNone(tcp) assert tcp is not None @@ -316,9 +302,9 @@ def test_tcp_helpers_cover_ipv6_and_data_payload_fallback(self) -> None: self.assertEqual(flow.dst, ip_address('2001:db8::20')) self.assertTrue(flow.syn) self.assertTrue(flow.fin) - self.assertEqual(flow.frame['ETHERNET']['FakeTCPIPv6']['TCP']['sport'], 2345) + self.assertEqual(flow.frame['ETHERNET']['IP6']['TCP']['sport'], 2345) - def test_ipv6_header_length_and_reassembly_with_fragment_fake(self) -> None: + def test_ipv6_header_length_and_reassembly_of_a_fragment(self) -> None: from pcapkit.const.reg.transtype import TransType from pcapkit.toolkit import dpkt as toolkit @@ -326,7 +312,8 @@ def test_ipv6_header_length_and_reassembly_with_fragment_fake(self) -> None: ipv6 = FakeIPv6(frag) self.assertEqual(toolkit.ipv6_hdr_len(ipv6), 48) - packet = types.SimpleNamespace(ip6=ipv6) + # the same fields on the wire, as the engine hands them over + packet = _fake_fragment_frame() reassembled = toolkit.ipv6_reassembly(packet, 0.0, count=5) self.assertIsNotNone(reassembled) assert reassembled is not None @@ -350,6 +337,8 @@ def test_ipv6_header_length_and_reassembly_with_fragment_fake(self) -> None: self.assertIsNone(toolkit.ipv6_reassembly(types.SimpleNamespace(), 0.0, count=1)) self.assertIsNone(toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=FakeIPv6()), 0.0, count=1)) + self.assertIsNone(toolkit.ipv6_reassembly(_ipv6_frame(b'PAYLOAD', 253, src='2001:db8::1', + dst='2001:db8::2'), 0.0, count=1)) # --------------------------------------------------------------------------- From 89b6103e3dc6428b297484f746eca310d454df35 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 13 Oct 2026 13:27:08 -0400 Subject: [PATCH 2/4] fix(engines): the dpkt engine attaches each frame's link type (#1626) A frame dpkt cannot decode is kept as raw octets, which do not say what link layer they start with, so the adapters could hand over nothing for it. The engine now stashes the link type beside the timestamp and the record octets, and the adapters read the frame by it. Part of #1626 --- pcapkit/foundation/engines/dpkt.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pcapkit/foundation/engines/dpkt.py b/pcapkit/foundation/engines/dpkt.py index e49b1ac33e..1ce352747f 100644 --- a/pcapkit/foundation/engines/dpkt.py +++ b/pcapkit/foundation/engines/dpkt.py @@ -476,9 +476,9 @@ def read_frame(self) -> 'DPKTPacket': for more operational information. """ - from pcapkit.toolkit.dpkt import (attach_buffer, attach_timestamp, ipv4_reassembly, - ipv6_reassembly, packet2dict, tcp_reassembly, - tcp_traceflow) + from pcapkit.toolkit.dpkt import (attach_buffer, attach_linktype, attach_timestamp, + ipv4_reassembly, ipv6_reassembly, packet2dict, + tcp_reassembly, tcp_traceflow) ext = self._extractor reader = self._extmp @@ -514,6 +514,9 @@ def read_frame(self) -> 'DPKTPacket': # it again recomputes zeroed checksums and lengths into it, so keep those # too for the adapters to slice the wire octets out of. attach_buffer(packet, pkt) + # And a frame kept as raw data does not say what link layer its octets + # start with, which the adapters read them by (#1626). + attach_linktype(packet, linktype) # verbose output ext._frnum += 1 From a3858ac79c47ab7250275f6ed8836030aee95389 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 13 Oct 2026 13:27:08 -0400 Subject: [PATCH 3/4] test(engines): drop the dpkt extension-chain gap #1609 closes dpkt's adapters now stop the chain at EXTENSION_HEADER_LIMIT, so the ipv6-33 and ipv6-1000 dpkt cases agree with default, and a gap that no longer fails is an error in the harness. Part of #1609 --- .../engines/test_ext_chain_limit_agreement_1604_unit.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/foundation/engines/test_ext_chain_limit_agreement_1604_unit.py b/tests/foundation/engines/test_ext_chain_limit_agreement_1604_unit.py index abd1c57958..60c51062d5 100644 --- a/tests/foundation/engines/test_ext_chain_limit_agreement_1604_unit.py +++ b/tests/foundation/engines/test_ext_chain_limit_agreement_1604_unit.py @@ -13,11 +13,12 @@ ``pypcapfile`` walks the chain itself and stops at the same depth. It leaves TCP over IPv6 out, so it is compared over IPv4 only, and ``declined`` checks that it warns of leaving out TCP over IPv6 exactly where ``default`` finds some. It -imports on Python 3.11 only, and is skipped elsewhere. +imports on Python 3.11 only, and is skipped elsewhere. ``dpkt``'s adapters walk +the chain off the octets, and stop at the same depth too (#1609). The other parsers walk the chain themselves, and cannot be told where to stop, so where they find the TCP past the limit is recorded in :attr:`KNOWN_FAILURES` -under #1609 rather than hidden. Measured on dpkt 1.9.8, scapy 2.7.0 and tshark 4.6.9. +under #1609 rather than hidden. Measured on scapy 2.7.0 and tshark 4.6.9. Every capture is built in a temporary directory, so no fixture is read. @@ -155,9 +156,6 @@ class TestExtensionHeaderChainAgreement(harness.RoundTripBase): STATUSES = ('ERROR', 'INPUTS', 'DECLINED') KNOWN_FAILURES = ( - Gap(1609, 'dpkt walks the whole extension header chain in a loop (dpkt.ip6.IP6.unpack), ' - 'so finds the TCP past EXTENSION_HEADER_LIMIT that the default engine leaves raw', - 'INPUTS', 'extra, default has none', ('ipv6-33/dpkt/*', 'ipv6-1000/dpkt/*')), Gap(1609, 'scapy dissects each extension header behind the last, with no limit of its own, ' 'so finds the TCP past EXTENSION_HEADER_LIMIT; it gives up near 245 layers at the ' 'default recursion limit, and so agrees at 1,000', From c874bfb732fb22b961c0133b004c2afceeea77b4 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 13 Oct 2026 13:27:08 -0400 Subject: [PATCH 4/4] test(toolkit): read dpkt's AttributeError off dpkt, and the TCP behind it (#1626) The test asserted _decode_ip, which the dpkt walk no longer needs. It now asserts dpkt's own decode raises, and that the TCP in the frame is read, as the default engine reads it. Part of #1626 --- .../test_tcp_carrier_walk_1606_1617_unit.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/toolkit/test_tcp_carrier_walk_1606_1617_unit.py b/tests/toolkit/test_tcp_carrier_walk_1606_1617_unit.py index f89b8a3890..2a6d19b522 100644 --- a/tests/toolkit/test_tcp_carrier_walk_1606_1617_unit.py +++ b/tests/toolkit/test_tcp_carrier_walk_1606_1617_unit.py @@ -274,18 +274,18 @@ def test_a_dpkt_packet_built_by_hand_is_walked_too(self) -> None: def test_a_tunnel_dpkt_cannot_decode_ends_no_extraction(self) -> None: # dpkt 1.9.8 reads ``frag_off`` from whatever header follows an IPv6 # Fragment header (#1351), so the IPv6 packet behind this Hop-by-Hop - # header raises AttributeError. The tunnel is left undecoded, as the - # engine leaves a frame it cannot decode, and the next frame is read. - # Whether TCP is read in it, as the default engine does, is #1626. + # header raises AttributeError. The next frame is still read, and the + # TCP in this one is read off the octets, as the default engine reads + # it (#1626). if not has('dpkt'): self.skipTest('dpkt not installed') - from pcapkit.toolkit import dpkt as toolkit + import dpkt inner = v6(fragment(0) + options(6) + tcp(1), 44) - self.assertIsNone(toolkit._decode_ip(41, inner)) # pylint: disable=protected-access + with self.assertRaises(AttributeError): + dpkt.ip6.IP6(inner) seen = read(self, 'dpkt', [eth(v4(options(41) + inner, 0)), eth(v4(tcp(2), 6))]) - self.assertIn(2, seen['tcp']) - self.assertIn(2, seen['trace']) + self.assertEqual((sorted(seen['tcp']), sorted(seen['trace'])), ([1, 2], [1, 2])) def test_a_dpkt_payload_that_is_no_packet_is_leaf_data(self) -> None: # dpkt decodes IPv4 protocol 0 as IPv4, so this Hop-by-Hop header and the