From 2e1a1c0187edd98e4647f53c20205deea0dc332f Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 13 Oct 2026 17:50:07 -0400 Subject: [PATCH] fix(toolkit): pyshark reads frames to the 64-layer bound and past tshark's 802.1Q limit, and keys its fallback by the header (#1609, #1671) - #1671: where a frame's octets cannot be had, _from_layers read tshark's ip.dst unguarded. tshark reports none behind a route option whose data is no whole number of addresses, so AttributeError ended extract(); behind a live LSRR or SSRR it is the route's last hop. _header_addresses now takes the header's destination (ip.cur_rt there) and reads no segment where tshark reports no address, as the default engine rejects such an option. - #1609: _from_octets followed IP-in-IP, AH and extension headers past layer 64. _network_start now gives the IP layer's place in the frame, and the walk goes through pypcapfile's _walk_upper_layer, which stops at FRAME_LAYER_LIMIT. - #1609: tshark dissects at most 20 802.1Q tags, so frames behind more were never read from their octets. Such a frame now asks for them (_stops_in_vlan_tags; OCTETS_FILTER gains vlan). New tests/toolkit/test_pyshark_fallback_and_layer_bound_1609_1671_unit.py: all 6 tests fail on main and pass here, on 3.11 with tshark 4.6.9. Closes #1609 Closes #1671 --- pcapkit/foundation/engines/pyshark.py | 13 +- pcapkit/toolkit/pyshark.py | 183 +++++++--- ...test_pyshark_encapsulated_tcp_1639_unit.py | 3 +- ...fallback_and_layer_bound_1609_1671_unit.py | 327 ++++++++++++++++++ ...yshark_ipv4_extension_headers_1625_unit.py | 28 +- 5 files changed, 484 insertions(+), 70 deletions(-) create mode 100644 tests/toolkit/test_pyshark_fallback_and_layer_bound_1609_1671_unit.py diff --git a/pcapkit/foundation/engines/pyshark.py b/pcapkit/foundation/engines/pyshark.py index 593a322ab3..eb65ae7698 100644 --- a/pcapkit/foundation/engines/pyshark.py +++ b/pcapkit/foundation/engines/pyshark.py @@ -566,8 +566,9 @@ def _captured(self, packet: 'PySharkPacket') -> 'Optional[Captured]': :func:`~pcapkit.toolkit.pyshark.tcp_traceflow` asks for one of a frame :data:`~pcapkit.toolkit.pyshark.OCTETS_FILTER` matches only: one tshark - dissects TCP in (:issue:`1639`), or one with an AH or IPv6 extension - header in IPv4 (:issue:`1625`). + dissects TCP in (:issue:`1639`), one with an AH or IPv6 extension + header in IPv4 (:issue:`1625`), or one with more 802.1Q tags than + tshark dissects (:issue:`1609`). Its link type is the one the default engine dissects it by, read from the capture file (see :func:`_linktypes`) for the frame of this engine's @@ -615,15 +616,15 @@ def _captured(self, packet: 'PySharkPacket') -> 'Optional[Captured]': if self._exraw_failed: return None frnum = self._extractor._frnum - source = ('the capture file, read for the link type of a frame with an AH or IPv6 extension ' - 'header in IPv4, or with TCP,') + frames = ('a frame with an AH or IPv6 extension header in IPv4, with TCP, or with more 802.1Q tags ' + 'than tshark dissects,') + source = f'the capture file, read for the link type of {frames}' try: linktype = self._read_linktype(frnum) if isinstance(linktype, str): reason = linktype else: - source = ('the second tshark pass, which reads the captured octets of a frame with an AH ' - 'or IPv6 extension header in IPv4, or with TCP,') + source = f'the second tshark pass, which reads the captured octets of {frames}' octets = self._read_octets(int(packet.number)) if isinstance(octets, bytes): return Captured(linktype, octets) diff --git a/pcapkit/toolkit/pyshark.py b/pcapkit/toolkit/pyshark.py index a07eb5bca0..2c06c19ddf 100644 --- a/pcapkit/toolkit/pyshark.py +++ b/pcapkit/toolkit/pyshark.py @@ -25,9 +25,10 @@ from pcapkit.const.reg.transtype import TransType as Enum_TransType from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet from pcapkit.protocols.link.loopback import family_byteorder +from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT from pcapkit.toolkit.pypcapfile import (IP_TUNNELS, IPV4_EXTENSION_HEADERS, IPV4_HEADER_LEN, TCP_MIN_HEADER_LEN, VLAN_TAG_LEN, VLAN_TAGS, - _default_accepts, _upper_layer) + _default_accepts, _walk_upper_layer) from pcapkit.utilities.exceptions import MissingKeyError, ProtocolError if TYPE_CHECKING: @@ -347,13 +348,15 @@ OCTETS_PROTOCOLS = IPV4_EXTENSION_HEADERS | {Enum_TransType.Shim6} #: tshark display filter matching every frame :func:`tcp_traceflow` asks the -#: captured octets of: one tshark dissects TCP in (see :func:`_traces`), or one -#: with an IPv4 header whose protocol is one of the :data:`OCTETS_PROTOCOLS`. -#: It matches every ``tcp`` layer and ``ip.proto`` field of a frame, so it -#: matches all of those frames and maybe others. The members are joined with -#: ``||`` rather than written as a set, as tshark 4.6.9 refuses a set whose -#: members are separated by spaces only. -OCTETS_FILTER = ' || '.join(['tcp', *(f'ip.proto == {int(protocol)}' for protocol in sorted(OCTETS_PROTOCOLS))]) +#: captured octets of: one tshark dissects TCP in (see :func:`_traces`), one +#: with an IPv4 header whose protocol is one of the :data:`OCTETS_PROTOCOLS`, or +#: one whose dissection stops in a stack of 802.1Q tags (see +#: :func:`_stops_in_vlan_tags`). It matches every ``tcp`` and ``vlan`` layer and +#: ``ip.proto`` field of a frame, so it matches all of those frames and maybe +#: others. The members are joined with ``||`` rather than written as a set, as +#: tshark 4.6.9 refuses a set whose members are separated by spaces only. +OCTETS_FILTER = ' || '.join(['tcp', *(f'ip.proto == {int(protocol)}' for protocol in sorted(OCTETS_PROTOCOLS)), + 'vlan']) #: Length of an Ethernet header: two MAC addresses and the EtherType. ETHERNET_HEADER_LEN = 14 @@ -598,6 +601,36 @@ def _has_fixed_header(tcp: 'Any') -> 'bool': return 'urgent_pointer' in tcp.field_names +def _header_addresses(ip: 'Any') -> 'Optional[tuple[IPv4Address | IPv6Address, IPv4Address | IPv6Address]]': + """Read the addresses of an IP header from tshark's dissection of it. + + Args: + ip: PyShark ``ip`` or ``ipv6`` layer. + + Returns: + The header's Source and Destination Address fields, as the default + engine keys a segment by them; or :data:`None` if tshark reports no + source or no destination address. + + Note: + Behind an LSRR or SSRR option whose route is not yet used up + (:rfc:`791#section-3.1`), tshark reports the route's last address as + ``ip.dst``, at its place in the option, and the header's as + ``ip.cur_rt``, which it reports nowhere else. Where the option's route + data is no whole number of addresses, it reports no ``ip.dst`` at all + ("Suboption would go past end of option"); the default engine's IPv4 + parser rejects such an option, so reads no TCP behind it. Both measured + with tshark 4.6.9 over every length and pointer of an LSRR and SSRR + option an IPv4 header holds (:issue:`1671`). + + """ + try: + src, dst = ip.src, ip.dst + except AttributeError: # what PyShark raises for a field tshark does not report + return None + return ipaddress.ip_address(str(src)), ipaddress.ip_address(str(getattr(ip, 'cur_rt', dst))) + + def _from_layers(packet: 'Packet') -> 'Optional[_Segment]': """Read the TCP segment of a PyShark packet from tshark's dissection of it. @@ -606,9 +639,9 @@ def _from_layers(packet: 'Packet') -> 'Optional[_Segment]': Returns: The packet's first TCP layer, if it has a whole fixed header (see - :func:`_has_fixed_header`), keyed by the IP layer that carries it -- in - an IP-in-IP tunnel, the innermost (see :func:`_carrier`); else - :data:`None`. + :func:`_has_fixed_header`), keyed by the addresses of the IP header + that carries it (see :func:`_header_addresses`) -- in an IP-in-IP + tunnel, the innermost (see :func:`_carrier`); else :data:`None`. Raises: ProtocolError: If a TCP flag field is not a boolean tshark spelling. @@ -617,9 +650,12 @@ def _from_layers(packet: 'Packet') -> 'Optional[_Segment]': ip, tcp = _carrier(packet) if ip is None or tcp is None or not _has_fixed_header(tcp): return None + addresses = _header_addresses(ip) + if addresses is None: + return None return _Segment( - src=ipaddress.ip_address(ip.src), - dst=ipaddress.ip_address(ip.dst), + src=addresses[0], + dst=addresses[1], srcport=int(tcp.srcport), dstport=int(tcp.dstport), seq=_absolute(tcp, 'seq'), @@ -637,13 +673,37 @@ def _traces(packet: 'Packet') -> 'bool': packet: PyShark packet. Returns: - Whether :func:`_from_layers` finds a segment in it, its flags unread. + Whether its first TCP layer has a whole fixed header (see + :func:`_has_fixed_header`) and an IP layer carries it (see + :func:`_carrier`). """ ip, tcp = _carrier(packet) return ip is not None and tcp is not None and _has_fixed_header(tcp) +def _stops_in_vlan_tags(packet: 'Packet') -> 'bool': + """Whether tshark's dissection of a PyShark packet stops in a stack of 802.1Q tags. + + Args: + packet: PyShark packet. + + Returns: + Whether it has a ``vlan`` layer and no ``ip`` or ``ipv6`` layer. + + Note: + tshark dissects at most 20 802.1Q tags of a frame, 802.1ad tags between + them or not, and nothing past the 21st ("Too many nested VLAN tags", + measured with tshark 4.6.9). The default engine dissects every one of + the :data:`~pcapkit.toolkit.pypcapfile.VLAN_TAGS` and the IP packet + behind them, up to :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` + layers of the frame (:issue:`1609`). + + """ + names = {layer.layer_name.lower() for layer in packet.layers} + return 'vlan' in names and not names & {'ip', 'ipv6'} + + def _extension_header_carrier(packet: 'Packet') -> 'Any': """Find the IPv4 layer of a PyShark packet whose protocol names an AH or IPv6 extension header. @@ -727,7 +787,7 @@ def _link_type(packet: 'Packet', frame: 'Any') -> 'Enum_LinkType': raise MissingKeyError('frame.encap_type=%s' % encap_type) from None -def _network_start(linktype: 'int', octets: 'bytes') -> 'Optional[tuple[int, int]]': +def _network_start(linktype: 'int', octets: 'bytes') -> 'Optional[tuple[int, int, int]]': """Find the outermost IP layer the default engine dissects in a frame, from its octets. Args: @@ -736,9 +796,12 @@ def _network_start(linktype: 'int', octets: 'bytes') -> 'Optional[tuple[int, int octets: The frame's captured octets. Returns: - The layer's IP version and where its header starts in ``octets``, or - :data:`None` if the default engine dissects no IP layer right after - the link layer. + The layer's IP version, where its header starts in ``octets``, and its + place in the frame, as the default engine counts layers towards + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`: ``1`` where the + frame is the IP packet, else ``2``, and one more for each VLAN tag, + which is a layer of its own (:issue:`1609`). :data:`None` if the + default engine dissects no IP layer right after the link layer. Note: The default engine dissects a frame by its link type @@ -766,26 +829,26 @@ def _network_start(linktype: 'int', octets: 'bytes') -> 'Optional[tuple[int, int if len(octets) < start + VLAN_TAG_LEN: return None ethertype, start = int.from_bytes(octets[start + 2:start + 4], 'big'), start + VLAN_TAG_LEN - version = ETHERTYPE_IP_VERSIONS.get(ethertype) + version, depth = ETHERTYPE_IP_VERSIONS.get(ethertype), 2 + (start - ETHERNET_HEADER_LEN) // VLAN_TAG_LEN elif linktype in (Enum_LinkType.NULL, Enum_LinkType.LOOP): if len(octets) < LOOPBACK_HEADER_LEN: return None - family, start = octets[:LOOPBACK_HEADER_LEN], LOOPBACK_HEADER_LEN + family, start, depth = octets[:LOOPBACK_HEADER_LEN], LOOPBACK_HEADER_LEN, 2 version = LOOPBACK_IP_VERSIONS.get(int.from_bytes(family, family_byteorder(family, linktype))) elif linktype == Enum_LinkType.RAW: if not octets: return None - version, start = octets[0] >> 4, 0 + version, start, depth = octets[0] >> 4, 0, 1 elif linktype in (Enum_LinkType.IPV4, Enum_LinkType.IPV6): - version, start = (4 if linktype == Enum_LinkType.IPV4 else 6), 0 + version, start, depth = (4 if linktype == Enum_LinkType.IPV4 else 6), 0, 1 else: return None if version is None or version not in (4, 6): return None - return version, start + return version, start, depth -def _network(packet: 'Packet', captured: 'Captured') -> 'Optional[tuple[int, bytes]]': +def _network(packet: 'Packet', captured: 'Captured') -> 'Optional[tuple[int, bytes, int]]': """Find the outermost IP packet the default engine dissects in a PyShark packet's frame. Args: @@ -793,16 +856,19 @@ def _network(packet: 'Packet', captured: 'Captured') -> 'Optional[tuple[int, byt captured: The frame, as the default engine reads it from the capture file. Returns: - The packet's IP version and its octets on, to the end of the frame, as - :func:`_network_start` finds it by the frame's link type, if tshark - dissected it too: the packet's first ``ip`` or ``ipv6`` layer is of that - version, and tshark reports its version field there, the layer's source - address in the header's Source Address field, and its destination - address in the octets where tshark reports that field (see below). - Else :data:`None`: the default engine dissects no IP packet right after - the link layer, or tshark dissected none there, e.g. as it reads an - IPv6 packet over link type ``IPV4`` (228) as IPv6, or dissected that - layer out of other octets, e.g. another frame's. + The packet's IP version, its octets on, to the end of the frame, and + its place in the frame, as :func:`_network_start` finds it by the + frame's link type, if tshark dissected it too: the packet's first + ``ip`` or ``ipv6`` layer is of that version, and tshark reports its + version field there, the layer's source address in the header's Source + Address field, and its destination address in the octets where tshark + reports that field (see below). Or if tshark dissected no IP layer, as + its dissection stopped in the VLAN tags before it (see + :func:`_stops_in_vlan_tags`). Else :data:`None`: the default engine + dissects no IP packet right after the link layer, or tshark dissected + none there, e.g. as it reads an IPv6 packet over link type ``IPV4`` + (228) as IPv6, or dissected that layer out of other octets, e.g. + another frame's. Note: The destination address is matched where tshark reports it, not in @@ -817,10 +883,12 @@ def _network(packet: 'Packet', captured: 'Captured') -> 'Optional[tuple[int, byt found = _network_start(linktype, octets) if found is None: return None - version, start = found + version, start, depth = found name = 'ip' if version == 4 else 'ipv6' ip = next((layer for layer in packet.layers if layer.layer_name.lower() in ('ip', 'ipv6')), None) + if ip is None and _stops_in_vlan_tags(packet): + return version, octets[start:], depth if ip is None or ip.layer_name.lower() != name: return None try: @@ -834,15 +902,17 @@ def _network(packet: 'Packet', captured: 'Captured') -> 'Optional[tuple[int, byt or octets[where:where + size] != source or target < start or octets[target:target + size] != destination): return None - return version, octets[start:] + return version, octets[start:], depth -def _from_octets(version: 'int', data: 'bytes') -> 'Optional[_Segment]': +def _from_octets(version: 'int', data: 'bytes', depth: 'int') -> 'Optional[_Segment]': """Read the TCP segment the default engine reads in an IP packet, from its octets. Args: version: IP version of the header ``data`` starts with, ``4`` or ``6``. data: Raw octets from the IP header on, to the end of the frame. + depth: The IP header's place in its frame, its first layer being ``1`` + (see :func:`_network_start`). Returns: The segment, keyed by the IP layer that carries it -- in an IP-in-IP @@ -855,11 +925,14 @@ def _from_octets(version: 'int', data: 'bytes') -> 'Optional[_Segment]': IPv4 parser accepts (see :func:`~pcapkit.toolkit.pypcapfile._default_accepts`), and each IP packet is followed past its headers by - :func:`~pcapkit.toolkit.pypcapfile._upper_layer`: an AH or IPv6 + :func:`~pcapkit.toolkit.pypcapfile._walk_upper_layer`: an AH or IPv6 extension header in IPv4 by the default engine's own parser, and the extension headers of IPv6, up to :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` of - them in a row; none in a later fragment. A TCP header must be at least + them in a row; none in a later fragment. No layer is read past + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the + frame, each IP header, AH and IPv6 extension header one of them + (:issue:`1609`). A TCP header must be at least :data:`~pcapkit.toolkit.pypcapfile.TCP_MIN_HEADER_LEN` octets long, as its Data Offset gives it, with that many captured. @@ -877,18 +950,20 @@ def _from_octets(version: 'int', data: 'bytes') -> 'Optional[_Segment]': """ while True: + if depth > FRAME_LAYER_LIMIT: + return None if version == 4 and len(data) >= IPV4_HEADER_LEN: ihl = (data[0] & 0x0F) * 4 if IPV4_HEADER_LEN < ihl <= len(data) and not _default_accepts('IPv4', data[:ihl]): return None - upper = _upper_layer(version, data) + upper = _walk_upper_layer(version, data, depth) if upper is None: return None if upper[0] not in IP_TUNNELS: break - version, data = IP_TUNNELS[upper[0]], upper[1] + version, data, depth = IP_TUNNELS[upper[0]], upper[1], upper[2] - protocol, segment = upper + protocol, segment, _ = upper if protocol != Enum_TransType.TCP or len(segment) < TCP_MIN_HEADER_LEN: return None header = (segment[12] >> 4) * 4 # the Data Offset, in octets @@ -923,9 +998,10 @@ def _segment(packet: 'Packet', captured: 'Optional[Callable[[Packet], Optional[C :func:`tcp_traceflow`). Returns: - If tshark's dissection has a segment (see :func:`_traces`), or an IPv4 + If tshark's dissection has a segment (see :func:`_traces`), an IPv4 layer whose protocol names an AH or IPv6 extension header (see - :func:`_extension_header_carrier`), the segment read from the octets of + :func:`_extension_header_carrier`), or stops in a stack of 802.1Q tags + (see :func:`_stops_in_vlan_tags`), the segment read from the octets of the frame's outermost IP packet on (see :func:`_network` and :func:`_from_octets`), or :data:`None` where the default engine dissects no IP packet after the frame's link layer or tshark dissects @@ -950,12 +1026,16 @@ def _segment(packet: 'Packet', captured: 'Optional[Callable[[Packet], Optional[C accepts it, and none behind Shim6 (140), whose parser refuses IPv4 (:issue:`1625`). Reading the frame from its link layer on, as the default engine does, stops where it stops: at a link type, EtherType or - tunnel it does not dissect, or past + tunnel it does not dissect, past :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` - extension headers in a row of an IPv6 packet. + extension headers in a row of an IPv6 packet, or past + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the + frame, which tshark reads on past; and it reads on past the 20 802.1Q + tags of a frame tshark stops at (:issue:`1609`). """ - if captured is not None and (_traces(packet) or _extension_header_carrier(packet) is not None): + if captured is not None and (_traces(packet) or _extension_header_carrier(packet) is not None + or _stops_in_vlan_tags(packet)): frame = captured(packet) if frame is not None: network = _network(packet, frame) @@ -987,12 +1067,15 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1, * If the ``packet`` can be used for TCP flow tracing. A packet can be reassembled if it contains TCP layer with a whole fixed header (see - :func:`_has_fixed_header`), keyed by the IP layer that carries it -- in an - IP-in-IP tunnel, the innermost (see :func:`_carrier`). Where + :func:`_has_fixed_header`), keyed by the addresses of the IP header that + carries it (see :func:`_header_addresses`) -- in an IP-in-IP tunnel, the + innermost (see :func:`_carrier`). Where ``captured`` has the frame, it is instead the TCP the default engine reads in the frame's octets, by the frame's link type in the capture file, and none where the default engine reads none: behind a link - type, EtherType or tunnel it does not dissect (see :func:`_segment`). + type, EtherType or tunnel it does not dissect, or past + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the + frame (see :func:`_segment`). * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`. diff --git a/tests/toolkit/test_pyshark_encapsulated_tcp_1639_unit.py b/tests/toolkit/test_pyshark_encapsulated_tcp_1639_unit.py index a93884d612..53a26b2db0 100644 --- a/tests/toolkit/test_pyshark_encapsulated_tcp_1639_unit.py +++ b/tests/toolkit/test_pyshark_encapsulated_tcp_1639_unit.py @@ -396,7 +396,8 @@ def test_frames_the_second_pass_cannot_read_are_traced_as_tshark_dissects_them(s messages = [call.args[0] for call in warn.call_args_list if call.args[1] is PySharkWarning] self.assertEqual(len(messages), 1, messages) self.assertIn('Frame 1: the second tshark pass, which reads the captured octets of a frame with an AH ' - 'or IPv6 extension header in IPv4, or with TCP,', messages[0]) + 'or IPv6 extension header in IPv4, with TCP, or with more 802.1Q tags than tshark dissects,', + messages[0]) class FakeField(str): diff --git a/tests/toolkit/test_pyshark_fallback_and_layer_bound_1609_1671_unit.py b/tests/toolkit/test_pyshark_fallback_and_layer_bound_1609_1671_unit.py new file mode 100644 index 0000000000..edf73e2a13 --- /dev/null +++ b/tests/toolkit/test_pyshark_fallback_and_layer_bound_1609_1671_unit.py @@ -0,0 +1,327 @@ +# -*- coding: utf-8 -*- +"""pyshark keys a segment by its header's addresses, and reads within the default engine's bounds. C.f. #1609, #1671. + +Where a frame's octets cannot be had, pyshark traces the TCP tshark dissects. +It took tshark's ``ip.dst`` for the destination there. tshark reports none +behind an LSRR or SSRR option whose route data is no whole number of +addresses, so a bare :exc:`AttributeError` ended the extraction; and behind a +route not yet used up, ``ip.dst`` is the route's last address, so the segment +was keyed by that hop. It now takes the header's destination, tshark's +``ip.cur_rt`` there, and reads no segment where tshark reports no destination, +as the default engine's parser rejects such an option (#1671). + +Read from its octets, a frame was followed through IP-in-IP, AH and extension +headers past its 64th layer, where the default engine dissects nothing. And a +frame tshark stops in was not read at all: tshark dissects no more than 20 +802.1Q tags of a frame. pyshark now counts the layers as the default engine +does, and reads a frame tshark stops in a stack of 802.1Q tags from its octets +too (#1609). + +Every capture is built here, so the module is in the unit tier. The tests that +run tshark skip unless the pyshark engine runs, which needs ``pyshark``, the +``tshark`` binary and Python 3.13 or older; the rest use fakes. + +""" +from __future__ import annotations + +import ipaddress +import struct +import unittest +from typing import TYPE_CHECKING +from unittest import mock + +from tests._support import reimport_once_per_class +from tests.foundation import _roundtrip as wire +from tests.toolkit.test_pyshark_encapsulated_tcp_1639_unit import (BAD_TIMESTAMP, HAS_RUNTIME, + HOP_1, HOP_2, LSRR, FakeField, + FakeLayer, build, eth, + fake_packet, ip_layer, read, + route, tcp, tcp_layer, v4, v6, + vlan) +from tests.toolkit.test_tcp_carrier_walk_1606_1617_unit import ah, options + +if TYPE_CHECKING: + from typing import Any, Callable + +#: An 802.1Q and an 802.1ad tag's TPID. +C_TAG, S_TAG = 0x8100, 0x88A8 + + +def option(kind: int, length: int, pointer: int, data: bytes) -> bytes: + """An IPv4 option of ``kind`` claiming ``length`` octets, then ``pointer`` and ``data``, padded to 32 bits.""" + raw = bytes([kind, length, pointer]) + data + return raw + bytes(-len(raw) % 4) + + +#: An LSRR option of length 9, whose route data is no whole number of addresses. +LSRR_9 = option(0x83, 9, 4, HOP_1 + b'\x01\x02') + + +def nest(packet: bytes, count: int, outer: int = 4) -> bytes: + """``packet`` tunnelled in ``count`` IPv4 (``outer`` 4) or IPv6 (6) headers.""" + for _ in range(count): + proto = 41 if packet[0] >> 4 == 6 else 4 + packet = v4(packet, proto) if outer == 4 else v6(packet, proto) + return packet + + +def pairs(count: int, ipv6: bool, number: int) -> bytes: + """TCP behind ``count`` IPv4 packets, each with an AH, or IPv6 packets, each with a Hop-by-Hop header.""" + packet, nxt = tcp(number), 6 + for _ in range(count): + packet = v6(options(nxt) + packet, 0) if ipv6 else v4(ah(nxt) + packet, 51) + nxt = 41 if ipv6 else 4 + return packet + + +def chain(count: int, nxt: int = 6) -> bytes: + """``count`` Destination Options headers in a row, the last naming ``nxt``.""" + return b''.join(bytes([60 if index < count - 1 else nxt, 0, 1, 4, 0, 0, 0, 0]) for index in range(count)) + + +#: Ethernet frames carrying TCP behind an IPv4 route option, each with whether +#: the default engine traces it, by a name for it, as +#: :data:`~tests.toolkit.test_pyshark_encapsulated_tcp_1639_unit.ENCAPSULATION_CASES`. +#: tshark reports no ``ip.dst`` behind each malformed route, and the last hop +#: as ``ip.dst`` behind each route not used up. +ROUTE_CASES = { + 'control': (True, lambda n: eth(v4(tcp(n)))), + 'lsrr of length 9': (False, lambda n: eth(v4(tcp(n), opts=LSRR_9))), + 'ssrr of length 9': (False, lambda n: eth(v4(tcp(n), opts=option(0x89, 9, 4, HOP_1 + b'\x01\x02')))), + 'lsrr of length 9, one hop used': (False, lambda n: eth(v4(tcp(n), opts=option(0x83, 9, 8, HOP_1 + b'\x01\x02')))), + 'lsrr of length 5': (False, lambda n: eth(v4(tcp(n), opts=option(0x83, 5, 4, b'\x01\x02')))), + 'lsrr of length 13': (False, lambda n: eth(v4(tcp(n), opts=option(0x83, 13, 4, HOP_1 + bytes(6))))), + 'lsrr of length 9, inner': (False, lambda n: eth(v4(v4(tcp(n), inner=True, opts=LSRR_9), 4))), + 'lsrr': (True, lambda n: eth(v4(tcp(n), opts=LSRR))), + 'ssrr': (True, lambda n: eth(v4(tcp(n), opts=route(0x89, 4, HOP_1)))), + 'lsrr, two hops, one used': (True, lambda n: eth(v4(tcp(n), opts=route(0x83, 8, HOP_1, HOP_2)))), + 'lsrr, used up': (True, lambda n: eth(v4(tcp(n), opts=route(0x83, 8, HOP_1)))), + 'record route': (True, lambda n: eth(v4(tcp(n), opts=route(0x07, 8, HOP_1)))), + 'lsrr, inner': (True, lambda n: eth(v4(v4(tcp(n), inner=True, opts=LSRR), 4))), + 'lsrr, 4in4': (True, lambda n: eth(v4(v4(tcp(n), inner=True), 4, opts=LSRR))), + 'lsrr, 6in4': (True, lambda n: eth(v4(v6(tcp(n), inner=True), 41, opts=LSRR))), + 'control, ipv6': (True, lambda n: eth(v6(tcp(n)))), + 'control, after the rest': (True, lambda n: eth(v4(tcp(n)))), +} + +#: Ethernet frames carrying TCP around the 64-layer bound of a frame, Ethernet +#: being layer 1, and each VLAN tag, IP header, AH and IPv6 extension header one +#: more; by a name for each, with whether the default engine traces it. tshark +#: traces every one but those behind more than 20 802.1Q tags. +BOUND_CASES = { + 'tcp in 62 ipv4 (layer 64)': (True, lambda n: eth(nest(v4(tcp(n)), 61))), + 'tcp in 63 ipv4 (layer 65)': (False, lambda n: eth(nest(v4(tcp(n)), 62))), + 'tcp in 100 ipv4': (False, lambda n: eth(nest(v4(tcp(n)), 99))), + 'tcp in 62 ipv6 (layer 64)': (True, lambda n: eth(nest(v6(tcp(n)), 61, 6))), + 'tcp in 63 ipv6 (layer 65)': (False, lambda n: eth(nest(v6(tcp(n)), 62, 6))), + 'tcp in ipv4 in 61 ipv6 (layer 64)': (True, lambda n: eth(nest(v4(tcp(n)), 61, 6))), + 'tcp in ipv4 in 62 ipv6 (layer 65)': (False, lambda n: eth(nest(v4(tcp(n)), 62, 6))), + 'tcp behind 31 (ipv4, ah) pairs (layer 64)': (True, lambda n: eth(pairs(31, False, n))), + 'tcp behind 32 (ipv4, ah) pairs (layer 66)': (False, lambda n: eth(pairs(32, False, n))), + 'tcp behind 31 (ipv6, hop-by-hop) pairs (layer 64)': (True, lambda n: eth(pairs(31, True, n))), + 'tcp behind 32 (ipv6, hop-by-hop) pairs (layer 66)': (False, lambda n: eth(pairs(32, True, n))), + 'tcp behind 61 ipv4 and a hop-by-hop header (layer 64)': ( + True, lambda n: eth(nest(v4(options(6) + tcp(n), 0), 60))), + 'tcp behind 62 ipv4 and a hop-by-hop header (layer 65)': ( + False, lambda n: eth(nest(v4(options(6) + tcp(n), 0), 61))), + 'tcp behind 50 ipv4, ipv6 and 11 extension headers (layer 64)': ( + True, lambda n: eth(nest(v6(chain(11) + tcp(n), 60), 50))), + 'tcp behind 50 ipv4, ipv6 and 12 extension headers (layer 65)': ( + False, lambda n: eth(nest(v6(chain(12) + tcp(n), 60), 50))), + 'tcp behind 61 802.1ad tags (layer 64)': (True, lambda n: vlan(v4(tcp(n)), *[S_TAG] * 61)), + 'tcp behind 62 802.1ad tags (layer 65)': (False, lambda n: vlan(v4(tcp(n)), *[S_TAG] * 62)), + # tshark dissects at most 20 802.1Q tags of a frame + 'tcp behind 20 802.1q tags': (True, lambda n: vlan(v4(tcp(n)), *[C_TAG] * 20)), + 'tcp behind 21 802.1q tags': (True, lambda n: vlan(v4(tcp(n)), *[C_TAG] * 21)), + 'tcp behind 30 802.1q tags, ipv6': (True, lambda n: vlan(v6(tcp(n)), *[C_TAG] * 30)), + 'tcp behind 42 802.1ad and 802.1q tags': (True, lambda n: vlan(v4(tcp(n)), *[S_TAG, C_TAG] * 21)), + 'tcp in 4in4 behind 30 802.1q tags': (True, lambda n: vlan(v4(v4(tcp(n), inner=True), 4), *[C_TAG] * 30)), + 'tcp behind 61 802.1q tags (layer 64)': (True, lambda n: vlan(v4(tcp(n)), *[C_TAG] * 61)), + 'tcp behind 62 802.1q tags (layer 65)': (False, lambda n: vlan(v4(tcp(n)), *[C_TAG] * 62)), + 'udp behind 30 802.1q tags': (False, lambda n: vlan(v4(wire.udp(b'data'), 17), *[C_TAG] * 30)), + 'tcp behind 30 802.1q tags, rejected ipv4 options': ( + False, lambda n: vlan(v4(tcp(n), opts=BAD_TIMESTAMP), *[C_TAG] * 30)), +} + +#: Frames of other link types carrying TCP around the bound, by a name for +#: each, with the link type and whether the default engine traces it. +LINK_BOUND_CASES = { + 'null, tcp in 62 ipv4 (layer 64)': (0, True, lambda n: b'\x02\x00\x00\x00' + nest(v4(tcp(n)), 61)), + 'null, tcp in 63 ipv4 (layer 65)': (0, False, lambda n: b'\x02\x00\x00\x00' + nest(v4(tcp(n)), 62)), + 'raw, tcp in 63 ipv4 (layer 64)': (101, True, lambda n: nest(v4(tcp(n)), 62)), + 'raw, tcp in 64 ipv4 (layer 65)': (101, False, lambda n: nest(v4(tcp(n)), 63)), +} + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class PySharkFallbackAndBoundTests(unittest.TestCase): + """The pyshark engine traces the TCP the default engine reads, as the cases say.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def agree(self, cases: 'dict[str, tuple[bool, Callable[[int], bytes]]]', linktype: int = 1) -> None: + frames, reads = build(cases) + capture = wire.pcap([(1, number, octets) for number, octets in enumerate(frames)], linktype=linktype) + names = dict(enumerate(cases, start=1)) + base = read(self, 'default', capture) + # what the default engine reads, as the cases say + self.assertEqual([names[number] for number in sorted(base['trace'])], [names[number] for number in reads]) + seen = read(self, 'pyshark', capture) + self.assertEqual({names[number]: seen['trace'].get(number) for number in names}, + {names[number]: base['trace'].get(number) for number in names}) + self.assertEqual(seen['flows'], base['flows']) + + def test_where_the_octets_cannot_be_had_a_route_option_ends_no_extraction(self) -> None: + # the second pass fails, so every frame is traced as tshark dissects + # it: the header's destination, and nothing where tshark reports none + with mock.patch('pcapkit.toolkit.pyshark.OCTETS_FILTER', 'tcp ||'): + self.agree(ROUTE_CASES) + self.agree({'raw, lsrr': (True, lambda n: v4(tcp(n), opts=LSRR)), + 'raw, lsrr of length 9': (False, lambda n: v4(tcp(n), opts=LSRR_9))}, 101) + + def test_nothing_is_traced_past_the_frame_layer_bound(self) -> None: + self.agree(BOUND_CASES) + for linktype in sorted({link for link, _, _ in LINK_BOUND_CASES.values()}): + with self.subTest(linktype=linktype): + self.agree({name: (reads, make) for name, (link, reads, make) in LINK_BOUND_CASES.items() + if link == linktype}, linktype) + + +def routed(octets: bytes, start: int, hop: bytes) -> FakeLayer: + """tshark's ``ip`` layer behind a route not used up: the header's destination as ``cur_rt``, ``hop`` as ``dst``.""" + layer = ip_layer(octets, start) + return FakeLayer('ip', **{**{name: getattr(layer, name) for name in layer.field_names}, 'cur_rt': layer.dst, + 'dst': FakeField(str(ipaddress.ip_address(hop)), start + 23, hop.hex())}) + + +def bare(layer: FakeLayer, *names: str) -> FakeLayer: + """``layer`` without its fields ``names``, as tshark reports a layer it does not finish.""" + return FakeLayer(layer.layer_name, **{name: getattr(layer, name) for name in layer.field_names + if name not in names}) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class ToolkitFallbackTests(unittest.TestCase): + """Where the frame cannot be had, :func:`tcp_traceflow` keys tshark's segment by the header; no tshark.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_a_header_tshark_reports_no_address_of_is_not_traced(self) -> None: + from pcapkit.toolkit import pyshark as toolkit + + octets = eth(v4(tcp(7), opts=LSRR_9)) + header = ip_layer(octets, 14) + for name, layer in (('no destination', bare(header, 'dst')), + ('no destination, the header\'s as cur_rt', FakeLayer('ip', **{ + **{field: getattr(header, field) for field in header.field_names if field != 'dst'}, + 'cur_rt': header.dst})), + ('no source', bare(header, 'src'))): + for captured in (None, lambda p: None): + with self.subTest(case=name, captured=captured): + packet = fake_packet('eth', '1', [layer, tcp_layer(octets, 46)]) + self.assertIsNone(toolkit.tcp_traceflow(packet, captured=captured)) + + def test_a_source_routed_segment_is_keyed_by_the_header_destination(self) -> None: + from pcapkit.toolkit import pyshark as toolkit + + plain = eth(v4(tcp(7), opts=LSRR)) + tunnel = eth(v4(v4(tcp(7), inner=True, opts=LSRR), 4)) # the inner header's addresses swapped + for name, packet, addresses in ( + ('lsrr', fake_packet('eth', '1', [routed(plain, 14, HOP_1), tcp_layer(plain, 42)]), + (wire.IPV4_A, wire.IPV4_B)), + ('lsrr, inner', fake_packet('eth', '1', [ip_layer(tunnel, 14), routed(tunnel, 34, HOP_1), + tcp_layer(tunnel, 62)]), (wire.IPV4_B, wire.IPV4_A)), + ): + for captured in (None, lambda p: None): + with self.subTest(case=name, captured=captured): + data = toolkit.tcp_traceflow(packet, captured=captured) + self.assertEqual((data.src, data.dst, data.srcport), + (*map(ipaddress.ip_address, addresses), 1007)) + # an IPv6 header's destination is tshark's ipv6.dst + octets = eth(v6(tcp(7))) + data = toolkit.tcp_traceflow(fake_packet('eth', '1', [ip_layer(octets, 14), tcp_layer(octets, 54)])) + self.assertEqual(data.dst, ipaddress.ip_address(wire.IPV6_B)) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class ToolkitBoundTests(unittest.TestCase): + """:func:`tcp_traceflow` reads a frame's octets within the default engine's bounds; no tshark.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def trace(self, linktype: int, octets: bytes, ip: int, at: int, link: str = 'eth') -> 'Any': + """Trace ``octets`` as tshark dissects them: the IP header at ``ip``, the TCP one at ``at``.""" + from pcapkit.toolkit import pyshark as toolkit + + packet = fake_packet(link, '1', [ip_layer(octets, ip), tcp_layer(octets, at)]) + return toolkit.tcp_traceflow(packet, captured=lambda p: toolkit.Captured(linktype, octets)) + + def test_no_tcp_is_read_past_the_64th_layer(self) -> None: + # Ethernet is layer 1, then each IP header, AH and extension header + reads = {} + for count, layer in ((61, 64), (62, 65)): + octets = eth(nest(v4(tcp(7)), count)) + reads[f'tcp in {count + 1} ipv4 (layer {layer})'] = self.trace(1, octets, 14, 14 + 20 * (count + 1)) + octets = nest(v4(tcp(7)), count + 1) + reads[f'raw, tcp in {count + 2} ipv4 (layer {layer})'] = self.trace(101, octets, 0, 20 * (count + 2), 'raw') + octets = eth(nest(v6(tcp(7)), count, 6)) + reads[f'tcp in {count + 1} ipv6 (layer {layer})'] = self.trace(1, octets, 14, 14 + 40 * (count + 1)) + for headers, layer in ((11, 64), (12, 65), (13, 66)): + octets = eth(nest(v6(chain(headers) + tcp(7), 60), 50)) + reads[f'tcp behind 50 ipv4, ipv6 and {headers} extension headers (layer {layer})'] = self.trace( + 1, octets, 14, 14 + 50 * 20 + 40 + 8 * headers) + for count, layer in ((31, 64), (32, 66)): + octets = eth(pairs(count, False, 7)) # each pair 44 octets: an IPv4 header and a 24-octet AH + reads[f'tcp behind {count} (ipv4, ah) pairs (layer {layer})'] = self.trace(1, octets, 14, + 14 + 44 * count) + self.assertEqual({name: data is not None for name, data in reads.items()}, { + 'tcp in 62 ipv4 (layer 64)': True, 'raw, tcp in 63 ipv4 (layer 64)': True, + 'tcp in 62 ipv6 (layer 64)': True, + 'tcp in 63 ipv4 (layer 65)': False, 'raw, tcp in 64 ipv4 (layer 65)': False, + 'tcp in 63 ipv6 (layer 65)': False, + 'tcp behind 50 ipv4, ipv6 and 11 extension headers (layer 64)': True, + 'tcp behind 50 ipv4, ipv6 and 12 extension headers (layer 65)': False, + 'tcp behind 50 ipv4, ipv6 and 13 extension headers (layer 66)': False, + 'tcp behind 31 (ipv4, ah) pairs (layer 64)': True, + 'tcp behind 32 (ipv4, ah) pairs (layer 66)': False, + }) + + def test_a_frame_tshark_stops_in_802_1q_tags_is_read_from_its_octets(self) -> None: + from pcapkit.toolkit import pyshark as toolkit + + # tshark dissects 21 vlan layers, the last with "Too many nested VLAN tags", and stops + stopped = [FakeLayer('vlan') for _ in range(21)] + reads = {} + for count, layer in ((30, 33), (61, 64), (62, 65)): + octets = vlan(v4(tcp(7)), *[C_TAG] * count) + asked = [] # type: list[Any] + packet = fake_packet('eth', '1', stopped) + data = toolkit.tcp_traceflow(packet, captured=lambda p, o=octets: asked.append(p) + or toolkit.Captured(1, o)) + self.assertEqual(asked, [packet]) + reads[f'tcp behind {count} 802.1q tags (layer {layer})'] = None if data is None else data.srcport + octets = wire.ethernet(struct.pack('>HH', 1, C_TAG) * 29 + struct.pack('>HH', 1, 0x0806) + bytes(28), C_TAG) + reads['arp behind 30 802.1q tags'] = toolkit.tcp_traceflow( + fake_packet('eth', '1', stopped), captured=lambda p: toolkit.Captured(1, octets)) + self.assertEqual(reads, {'tcp behind 30 802.1q tags (layer 33)': 1007, + 'tcp behind 61 802.1q tags (layer 64)': 1007, + 'tcp behind 62 802.1q tags (layer 65)': None, + 'arp behind 30 802.1q tags': None}) + + # a frame tshark dissects an IP layer behind its tags in is not asked + # for, unless it finds TCP there; and the second pass writes every + # frame with a VLAN tag + def refuse(packet: 'Any') -> 'Any': + raise AssertionError('octets asked for') + + octets = vlan(v4(wire.udp(b'data'), 17), C_TAG) + self.assertIsNone(toolkit.tcp_traceflow(fake_packet('eth', '1', [FakeLayer('vlan'), ip_layer(octets, 18), + FakeLayer('udp')]), captured=refuse)) + self.assertEqual(toolkit.OCTETS_FILTER.rsplit(' || ', maxsplit=1)[1], 'vlan') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/toolkit/test_pyshark_ipv4_extension_headers_1625_unit.py b/tests/toolkit/test_pyshark_ipv4_extension_headers_1625_unit.py index b8bea89b44..d28cd14895 100644 --- a/tests/toolkit/test_pyshark_ipv4_extension_headers_1625_unit.py +++ b/tests/toolkit/test_pyshark_ipv4_extension_headers_1625_unit.py @@ -634,7 +634,8 @@ def test_the_octets_are_read_as_the_default_engine_reads_them(self) -> None: version = {b'\x08\x00': 4, b'\x86\xdd': 6}[octets[12:14]] with self.subTest(case=name), warnings.catch_warnings(): warnings.simplefilter('ignore') # what the parsers warn of rejecting a header - segment = toolkit._from_octets(version, octets[14:]) # pylint: disable=protected-access + # the IP header is the frame's second layer, after Ethernet + segment = toolkit._from_octets(version, octets[14:], 2) # pylint: disable=protected-access record = base['trace'].get(number) if record is None: self.assertIsNone(segment) @@ -646,29 +647,30 @@ def test_the_octets_are_read_as_the_default_engine_reads_them(self) -> None: def test_the_outermost_ip_layer_is_found_as_the_default_engine_finds_it(self) -> None: from pcapkit.toolkit import pyshark as toolkit + # its version, where it starts, and its place in the frame (#1609) packet4, packet6 = v4(tcp(1), 6), v6(tcp(1), 6) for name, linktype, octets, found in ( - ('ethernet', 1, eth(packet4), (4, 14)), - ('ethernet, ipv6', 1, eth(packet6), (6, 14)), - ('vlan', 1, vlan(packet4, 0x8100), (4, 18)), - ('qinq', 1, vlan(packet4, 0x88A8, 0x8100), (4, 22)), + ('ethernet', 1, eth(packet4), (4, 14, 2)), + ('ethernet, ipv6', 1, eth(packet6), (6, 14, 2)), + ('vlan', 1, vlan(packet4, 0x8100), (4, 18, 3)), + ('qinq', 1, vlan(packet4, 0x88A8, 0x8100), (4, 22, 4)), ('ethernet, arp', 1, wire.ethernet(packet4, 0x0806), None), ('ethernet, mpls', 1, wire.ethernet(b'\x00\x01\x01\x40' + packet4, 0x8847), None), ('ethernet cut short', 1, eth(packet4)[:13], None), ('vlan tag cut short', 1, vlan(packet4, 0x8100)[:17], None), - ('null', 0, loopback(packet4, 2, 'little'), (4, 4)), - ('null, big-endian', 0, loopback(packet6, 28, 'big'), (6, 4)), + ('null', 0, loopback(packet4, 2, 'little'), (4, 4, 2)), + ('null, big-endian', 0, loopback(packet6, 28, 'big'), (6, 4, 2)), ('null, another family', 0, loopback(packet4, 7, 'little'), None), ('null cut short', 0, b'\x02\x00\x00', None), - ('loop', 108, loopback(packet6, 24, 'big'), (6, 4)), + ('loop', 108, loopback(packet6, 24, 'big'), (6, 4, 2)), # a LOOP family is in network byte order, never inferred ('loop, little-endian', 108, loopback(packet4, 2, 'little'), None), - ('raw', 101, packet4, (4, 0)), - ('raw, ipv6', 101, packet6, (6, 0)), + ('raw', 101, packet4, (4, 0, 1)), + ('raw, ipv6', 101, packet6, (6, 0, 1)), ('raw, another version', 101, b'\x55' + packet4[1:], None), ('raw, empty', 101, b'', None), - ('ipv4', 228, packet4, (4, 0)), - ('ipv6', 229, packet6, (6, 0)), + ('ipv4', 228, packet4, (4, 0, 1)), + ('ipv6', 229, packet6, (6, 0, 1)), ('linux sll', 113, sll(packet4), None), ): with self.subTest(case=name): @@ -810,7 +812,7 @@ def test_the_filter_names_every_extension_header(self) -> None: self.assertEqual(toolkit.OCTETS_FILTER, 'tcp || ip.proto == 0 || ip.proto == 43 || ip.proto == 44 || ' 'ip.proto == 51 || ip.proto == 60 || ip.proto == 135 || ' - 'ip.proto == 139 || ip.proto == 140') + 'ip.proto == 139 || ip.proto == 140 || vlan') self.assertEqual(len(IPV4_EXTENSION_HEADERS), 7) self.assertEqual(toolkit.OCTETS_PROTOCOLS - IPV4_EXTENSION_HEADERS, {140})