diff --git a/docs/source/pcapkit/protocols/internet/internet.rst b/docs/source/pcapkit/protocols/internet/internet.rst index a60a8b75c..ff7301969 100644 --- a/docs/source/pcapkit/protocols/internet/internet.rst +++ b/docs/source/pcapkit/protocols/internet/internet.rst @@ -37,8 +37,8 @@ A chain of IPv6 extension headers is dissected to at most a row, whether it follows IPv6 or another layer such as IPv4. What follows is kept as :class:`~pcapkit.protocols.misc.raw.Raw`, with one :exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1604`). The -``pypcapfile`` toolkit stops at the same depth. The ``dpkt``, ``scapy`` and -``pyshark`` engines leave the walk to their own parsers, which do not stop -there, so they may still find the upper layer past it (:issue:`1609`). +``pypcapfile``, ``dpkt`` and ``scapy`` toolkits stop at the same depth. The +``pyshark`` engine leaves the walk to its own parser, which does not stop +there, so it may still find the upper layer past it (:issue:`1609`). .. autodata:: pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT diff --git a/docs/source/pcapkit/protocols/protocol.rst b/docs/source/pcapkit/protocols/protocol.rst index 43a96eba6..844888870 100644 --- a/docs/source/pcapkit/protocols/protocol.rst +++ b/docs/source/pcapkit/protocols/protocol.rst @@ -116,8 +116,9 @@ there back out, as they do one the TCP parser rejected (:issue:`1518`). A protocol constructed from its own octets is not counted itself, so the count starts at the first layer it dissects. One chain of extension headers is also bounded on its own, by :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT`. -The ``dpkt``, ``scapy``, ``pyshark`` and ``pypcapfile`` engines dissect with -their own parsers, which do not apply this bound (:issue:`1609`). +The ``pyshark`` and ``pypcapfile`` engines dissect with their own parsers, +which do not apply this bound (:issue:`1609`). The ``dpkt`` and ``scapy`` +toolkits count the layers they walk as the default engine does, and stop at it. .. autodata:: pcapkit.protocols.protocol.FRAME_LAYER_LIMIT diff --git a/pcapkit/toolkit/scapy.py b/pcapkit/toolkit/scapy.py index 74e27b912..2261184b2 100644 --- a/pcapkit/toolkit/scapy.py +++ b/pcapkit/toolkit/scapy.py @@ -19,7 +19,7 @@ Several functions here import a `Scapy`_ layer module lazily, inside the function body -- :func:`ipv6_reassembly` needs - :class:`scapy.layers.inet6.IPv6ExtHdrFragment`, for instance. Those imports are + :class:`scapy.layers.inet6.IPv6`, for instance. Those imports are for the *classes* they name and nothing more. They must not be mistaken for how `Scapy`_'s layer registries get populated, even though they do populate them as a side effect, because by the time any of these functions runs the engine has @@ -37,8 +37,10 @@ """ import decimal import ipaddress -from typing import TYPE_CHECKING, cast +import struct +from typing import TYPE_CHECKING, NamedTuple, cast +from pcapkit.const.ipv6.extension_header import ExtensionHeader as Enum_ExtensionHeader from pcapkit.const.reg.linktype import LinkType as Enum_LinkType from pcapkit.const.reg.transtype import TransType as Enum_TransType from pcapkit.foundation.reassembly.data.ip import Packet as IP_Packet @@ -46,10 +48,12 @@ from pcapkit.foundation.traceflow.data.data import FrameRecord from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet from pcapkit.protocols.data.misc.pcap.frame import FrameInfo as Data_FrameInfo +from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT +from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length from pcapkit.toolkit.pypcapfile import (IP_TUNNELS, IPV4_EXTENSION_HEADERS, IPV4_HEADER_LEN, - IPV6_HEADER_LEN, TCP_MIN_HEADER_LEN, _default_accepts, - _past_extension_headers) + IPV6_EXTENSION_HEADERS, IPV6_HEADER_LEN, TCP_MIN_HEADER_LEN, + _default_accepts, _walk_extension_headers) from pcapkit.utilities.compat import ModuleNotFoundError # pylint: disable=redefined-builtin from pcapkit.utilities.exceptions import MissingKeyError, ModuleNotFound, stacklevel from pcapkit.utilities.warnings import ScapyWarning, warn @@ -63,10 +67,10 @@ if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address - from typing import Any, Iterator + from typing import Any, Iterator, Optional from scapy.layers.inet import IP, TCP - from scapy.layers.inet6 import IPv6, IPv6ExtHdrFragment + from scapy.layers.inet6 import IPv6 from scapy.packet import Packet __all__ = [ @@ -83,6 +87,12 @@ #: ``LINKTYPE_*`` number, by :func:`attach_linktype`. LINKTYPE_ATTR = '__pcapkit_linktype__' +#: The BSD loopback address families the default engine dissects IP behind, +#: each mapped to the IP version it carries, as +#: :class:`~pcapkit.protocols.link.loopback.Loopback` reads them (:issue:`1574`): +#: ``AF_INET``, and the ``AF_INET6`` of each of the BSDs. +LOOPBACK_FAMILIES = {2: 4, 24: 6, 28: 6, 30: 6} + def attach_resolution(packet: 'Packet', resolution: 'int') -> 'None': """Stash the timestamp resolution of a packet's capture on the packet. @@ -362,103 +372,313 @@ def _quoted(layer: 'Packet') -> 'bool': return isinstance(layer, (IPerror, IPerror6)) -def _extension_headers() -> 'tuple[type[Packet], ...]': - """The `Scapy`_ layers of an IPv6 extension header chain that the default engine reads past. +def _linktype(packet: 'Packet') -> 'Optional[int]': + """The data link type of the capture a Scapy packet was read from. - Returns: - The base class of `Scapy`_'s IPv6 Hop-by-Hop Options, Routing, Fragment - and Destination Options headers, and its AH. + Args: + packet: Scapy packet. - Note: - Of the IPv6 extension headers - (:class:`~pcapkit.const.ipv6.extension_header.ExtensionHeader`), the - default engine reads past the Hop-by-Hop Options, Routing, Fragment, - Destination Options, AH, Mobility, HIP and Shim6 headers. It stops at - ESP, whose Next Header field lies in the encrypted part of the packet - (:rfc:`4303`), and at codes 253 and 254, which have no parser. `Scapy`_ - dissects the ESP and Mobility headers but nothing behind them, nothing - but IP behind an AH, and neither HIP nor Shim6. + Returns: + The ``LINKTYPE_*`` number :class:`~pcapkit.foundation.engines.scapy.Scapy` + attached with :func:`attach_linktype`. A packet built by hand carries + none, so its first layer stands in for it: ``ETHERNET`` for an Ethernet + II or 802.3 frame, ``LOOP`` and ``NULL`` for the two kinds of BSD + loopback header `Scapy`_ has, and ``IPV4`` or ``IPV6`` for an IP + packet; and :data:`None` for any other, which no link type the default + engine dissects selects. """ - from scapy.layers.inet6 import _IPv6ExtHdr - from scapy.layers.ipsec import AH + linktype = getattr(packet, LINKTYPE_ATTR, None) + if linktype is not None: + return int(linktype) - return _IPv6ExtHdr, AH + from scapy.layers.inet import IP + from scapy.layers.inet6 import IPv6 + from scapy.layers.l2 import Dot3, Ether, Loopback, LoopbackOpenBSD + + if isinstance(packet, (Ether, Dot3)): + return Enum_LinkType.ETHERNET + if isinstance(packet, LoopbackOpenBSD): + return Enum_LinkType.LOOP + if isinstance(packet, Loopback): + return Enum_LinkType.NULL + if isinstance(packet, IP): + return Enum_LinkType.IPV4 + if isinstance(packet, IPv6): + return Enum_LinkType.IPV6 + return None -def _carries(layer: 'Packet', *, root: 'bool' = False) -> 'bool': - """Test if the default engine dissects what a Scapy layer carries, as `Scapy`_ does. +def _network(packet: 'Packet') -> 'Optional[tuple[Packet, int]]': + """Fetch the layer of a Scapy packet the default engine dissects behind its link layer. Args: - layer: A layer of a dissected Scapy packet. - root: Whether ``layer`` is the packet's first, the one the capture's - link type selects. + packet: Scapy packet. Returns: - For an IPv4 layer, whether the default engine accepts its header (see - :func:`_ipv4_accepted`) and it is no later fragment, whose data is a - slice of the datagram's payload with no upper-layer header in it - (:rfc:`791#section-3.2`, :issue:`1617`). For any other, whether it is - the first, an 802.1Q or 802.1ad tag, an IPv6 layer or an extension - header in its chain (see :func:`_extension_headers`). :data:`False` - for the datagram an ICMP or ICMPv6 error quotes (see :func:`_quoted`). + That layer and its place in the frame, the first layer being ``1`` (c.f. + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`), by the + packet's link type (see :func:`_linktype`): the payload of an Ethernet + frame, which may be anything `Scapy`_ dissects there; the packet itself + for a raw IP link type; and for a BSD loopback header the IPv4 or IPv6 + packet its address family names, as the default engine reads it (see + :data:`LOOPBACK_FAMILIES`). :data:`None` for any other link type, whose + frames the default engine leaves raw (:issue:`1650`), or a loopback + family it dissects nothing behind. Note: - The default engine dissects IP only where its dispatch tables send it: - an EtherType of the link layer or of a VLAN tag - (:attr:`Link.__proto__ `), - and a protocol number of IP or of an extension header - (:attr:`Internet.__proto__ - `). It leaves - raw the other layers `Scapy`_ dissects IP inside -- GRE, PPPoE, an - 802.3 frame's LLC/SNAP header, VXLAN, the PPP frame L2TPv2 carries, - and ICMP -- so it reads no IP, and no TCP, past one (:issue:`1636`). - The first layer is the one the capture's link type selects, and is - taken to carry whatever `Scapy`_ dissects in it. Where `Scapy`_ - dissects another layer before IP there -- PPP behind HDLC-like framing - (link types 9 and 50) or PPPoE (51), the LLC header of an 802.11 frame - (105), or the 802.11 frame behind a radiotap or PPI header (127, 192) - -- that layer is taken to carry nothing, as the default engine - dissects none of those link types. + The default engine dissects the first layer of the ``NULL``, + ``ETHERNET``, ``LOOP``, ``RAW``, ``IPV4`` and ``IPV6`` link types alone + (:attr:`Frame.__proto__ `), + and ``RAW`` as IPv4 or IPv6 by its version nibble, as `Scapy`_ does. + `Scapy`_ dissects IP straight behind the headers of many others, such as + ``LINUX_SLL``, ``LINUX_SLL2``, ``PPP`` and the ``DLT_RAW`` of 12. + + A loopback header's family is read as + :func:`~pcapkit.protocols.link.loopback.family_byteorder` reads it -- + ``LOOP``'s in network byte order, ``NULL``'s in the byte order its + octets show -- and the packet behind it is dissected from the octets + wherever `Scapy`_ dissects otherwise. `Scapy`_ binds IPv6 to the + ``AF_INET6`` of the host it runs on, and IPv4 to family ``0`` too, so + what it dissects there depends on the host. """ from scapy.layers.inet import IP from scapy.layers.inet6 import IPv6 - from scapy.layers.l2 import Dot1AD, Dot1Q - if _quoted(layer): - return False - if isinstance(layer, IP): - return not layer.frag and _ipv4_accepted(layer) - return root or isinstance(layer, (Dot1Q, Dot1AD, IPv6, *_extension_headers())) + linktype = _linktype(packet) + if linktype == Enum_LinkType.ETHERNET: + return packet.payload, 2 + if linktype in (Enum_LinkType.RAW, Enum_LinkType.IPV4, Enum_LinkType.IPV6): + return packet, 1 + if linktype not in (Enum_LinkType.NULL, Enum_LinkType.LOOP): + return None + from pcapkit.protocols.link.loopback import family_byteorder # isort:skip -def _dissected(packet: 'Packet') -> 'Iterator[Packet]': + octets = bytes(packet) + if len(octets) < 4: + return None + version = LOOPBACK_FAMILIES.get(int.from_bytes(octets[:4], family_byteorder(octets[:4], linktype))) + if version is None: + return None + layer = packet.payload # type: Packet + if not isinstance(layer, IP if version == 4 else IPv6): + data = octets[4:] + if len(data) < (IPV4_HEADER_LEN if version == 4 else IPV6_HEADER_LEN): + return None + layer = IP(data) if version == 4 else IPv6(data) + return layer, 2 + + +class _Chain(NamedTuple): + """The extension header chain of a Scapy IPv6 layer, as the default engine walks it.""" + + #: The octets of the IPv6 packet's payload, as far as its Payload Length and + #: every IP layer above it goes (see :func:`_within_datagrams`). + payload: 'bytes' + #: How many extension headers the default engine walks. + headers: 'int' + #: Where the first Fragment header it walks starts in :attr:`payload`, or + #: :data:`None` if it walks none. + fragment: 'Optional[int]' + #: The upper layer it dissects behind the chain: the `Scapy`_ layer, or one + #: dissected here from :attr:`payload`; or :data:`None` if it dissects no + #: TCP, IPv4 or IPv6 there. + upper: 'Optional[Packet]' + + +def _ipv6_chain(ipv6: 'IPv6', depth: 'int') -> '_Chain': + """Walk the extension header chain of a Scapy IPv6 layer as the default engine walks it. + + Args: + ipv6: IPv6 layer of a dissected Scapy packet. + depth: Its place in the frame (see :func:`_network`). + + Returns: + The chain, walked over the octets by the default engine's rules, as + :func:`pcapkit.toolkit.pypcapfile._upper_layer` walks it: the + :data:`~pcapkit.toolkit.pypcapfile.IPV6_EXTENSION_HEADERS`, each sized + by its own length field, up to + :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` of + them (:issue:`1604`) and as far as + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`, each header being + one layer of the frame (:issue:`1610`), and no further than a later + fragment's Fragment header (:rfc:`8200#section-4.5`) or a header cut + short. The upper layer is a TCP, IPv4 or IPv6 one inside the frame + bound, dissected from the octets where `Scapy`_ dissected something + else there. + + Note: + `Scapy`_ dissects the Hop-by-Hop Options, Routing, Fragment and + Destination Options headers, and AH, but no TCP and no IPv4 behind an + AH, and IPv4 where the Next Header is ``0``; the Mobility header but + nothing behind it; and neither HIP nor Shim6 (:issue:`1626`). Nor does + it stop at either bound (:issue:`1609`). Its layers are kept as far as + they are the ones the walk steps over, so the upper layer is its own + wherever it dissected the chain as the default engine does. + + """ + from scapy.layers.inet import IP, TCP + from scapy.layers.inet6 import IPv6, _IPv6ExtHdr + from scapy.layers.ipsec import AH + + head = bytes(ipv6)[:IPV6_HEADER_LEN] + payload = _within_datagrams(ipv6.payload) + nxt = head[6] if len(head) == IPV6_HEADER_LEN else None + offset, headers, fragment = 0, 0, None # type: int, int, Optional[int] + bound = min(EXTENSION_HEADER_LIMIT, FRAME_LAYER_LIMIT - depth) + # the headers Scapy dissects, as AH or one of its ``_IPv6ExtHdr`` layers + dissects = (Enum_ExtensionHeader.HOPOPT, Enum_ExtensionHeader.IPv6_Route, Enum_ExtensionHeader.IPv6_Frag, + Enum_ExtensionHeader.IPv6_Opts, Enum_ExtensionHeader.AH) + layer = ipv6.payload # type: Optional[Packet] # Scapy's, while it walks the chain alike + while nxt in IPV6_EXTENSION_HEADERS: + if headers == bound or len(payload) < offset + 8: + return _Chain(payload, headers, fragment, None) + headers += 1 + if nxt in dissects and isinstance(layer, AH if nxt == Enum_ExtensionHeader.AH else _IPv6ExtHdr): + layer = layer.payload + else: + layer = None + if nxt == Enum_ExtensionHeader.IPv6_Frag: # fixed length, second octet reserved + if fragment is None: + fragment = offset + if struct.unpack_from('!H', payload, offset + 2)[0] >> 3: # a later fragment + return _Chain(payload, headers, fragment, None) + length = 8 + elif nxt == Enum_ExtensionHeader.AH: # length in 4-octet units, less two + length = (payload[offset + 1] + 2) * 4 + else: # length in 8-octet units, less one + length = (payload[offset + 1] + 1) * 8 + nxt, offset = payload[offset], offset + length + + upper = None # type: Optional[Packet] + data = payload[offset:] + if depth + headers + 1 > FRAME_LAYER_LIMIT or len(payload) < offset: + pass + elif nxt == Enum_TransType.TCP: + if isinstance(layer, TCP): + upper = layer + elif len(data) >= TCP_MIN_HEADER_LEN: + upper = TCP(data) + elif nxt in IP_TUNNELS: + version = IP_TUNNELS[nxt] + if isinstance(layer, IP if version == 4 else IPv6) and not _quoted(layer): + upper = layer + elif len(data) >= (IPV4_HEADER_LEN if version == 4 else IPV6_HEADER_LEN): + upper = IP(data) if version == 4 else IPv6(data) + return _Chain(payload, headers, fragment, upper) + + +class _Layer(NamedTuple): + """A layer of a Scapy packet that the default engine dissects too.""" + + #: The `Scapy`_ layer, or one dissected here from the octets. + layer: 'Packet' + #: Its place in the frame, the first layer being ``1`` (c.f. + #: :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`). + depth: 'int' + #: For a TCP layer the default engine reads behind an AH or IPv6 extension + #: header in IPv4, its header and payload as that engine splits them (see + #: :func:`_tcp_behind_extension_headers`); else :data:`None`. + segment: 'Optional[tuple[bytes, bytes]]' = None + + +def _dissected(packet: 'Packet') -> 'Iterator[_Layer]': """Walk the layers of a Scapy packet that the default engine dissects too. Args: packet: Scapy packet. Yields: - Each layer of the packet from the first on, up to the first that does - not carry what follows it as the default engine dissects it (see - :func:`_carries`), and short of the datagram an ICMP or ICMPv6 error - quotes (see :func:`_quoted`). + The layer the default engine dissects behind the link layer (see + :func:`_network`), and each one on from it that it dissects as well, + up to the first that carries nothing it dissects: anything but an + 802.1Q or 802.1ad tag, IPv4 or IPv6; an IPv4 layer that is a later + fragment, whose data is a slice of the datagram's payload with no + upper-layer header in it (:rfc:`791#section-3.2`, :issue:`1617`). It + stops short of an IPv4 layer whose header the default engine rejects + (see :func:`_ipv4_accepted`), of the datagram an ICMP or ICMPv6 error + quotes (see :func:`_quoted`), and of + :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`, past which the + default engine dissects nothing (:issue:`1609`). + + An IPv6 layer is followed by the upper layer behind its extension + headers (see :func:`_ipv6_chain`), which are not yielded. An IPv4 layer + whose protocol names an AH or IPv6 extension header carries what the + default engine reads past those headers, by its own rules and up to + :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` of + them in a row (see + :func:`~pcapkit.toolkit.pypcapfile._walk_extension_headers`): a tunnel, + or a TCP segment, each dissected from the octets (:issue:`1606`). Note: + The default engine dissects IP only where its dispatch tables send it: + a link type, an EtherType of the link layer or of a VLAN tag + (:attr:`Link.__proto__ `), + and a protocol number of IP or of an extension header + (:attr:`Internet.__proto__ + `). It leaves + raw the other layers `Scapy`_ dissects IP inside -- GRE, PPPoE, an + 802.3 frame's LLC/SNAP header, VXLAN, the PPP frame L2TPv2 carries, + and ICMP -- so it reads no IP, and no TCP, past one (:issue:`1636`). + + Inside IPv4, `Scapy`_ dissects an AH alone of those headers, and behind + one only IPv6 and, for a Next Header of ``0``, IPv4, so the default + engine's rules are applied to the octets instead. It also dissects IPv6 + and AH inside a later fragment, binding neither to a zero fragment + offset as it binds TCP. + The payloads are walked, not ``getlayer()``, which searches the packets a field holds as well, such as the IPv6 packet a Redirected Header option carries (:rfc:`4861#section-4.6.3`). """ + from scapy.layers.inet import IP, TCP + from scapy.layers.inet6 import IPv6 + from scapy.layers.l2 import Dot1AD, Dot1Q from scapy.packet import NoPayload - layer = packet # type: Packet - while not isinstance(layer, NoPayload) and not _quoted(layer): - yield layer - if not _carries(layer, root=layer is packet): - return - layer = layer.payload + if (network := _network(packet)) is None: + return + layer, depth = network + while depth <= FRAME_LAYER_LIMIT and not isinstance(layer, NoPayload) and not _quoted(layer): + if isinstance(layer, IP): + if not _ipv4_accepted(layer): + return + yield _Layer(layer, depth) + if layer.frag: + return + if layer.proto not in IPV4_EXTENSION_HEADERS: + layer, depth = layer.payload, depth + 1 + continue + if (walked := _walk_extension_headers(layer.proto, _within_datagrams(layer.payload))) is None: + return + proto, data, hops = walked + depth += hops + 1 + if depth > FRAME_LAYER_LIMIT: + return + if proto == Enum_TransType.TCP: + if (segment := _tcp_behind_extension_headers(data)) is not None: + yield _Layer(TCP(data), depth, segment) + return + if proto not in IP_TUNNELS: + return + version = IP_TUNNELS[proto] + if len(data) < (IPV4_HEADER_LEN if version == 4 else IPV6_HEADER_LEN): + return + layer = IP(data) if version == 4 else IPv6(data) + elif isinstance(layer, IPv6): + yield _Layer(layer, depth) + chain = _ipv6_chain(layer, depth) + if chain.upper is None: + return + layer, depth = chain.upper, depth + chain.headers + 1 + else: + yield _Layer(layer, depth) + if not isinstance(layer, (Dot1Q, Dot1AD)): + return + layer, depth = layer.payload, depth + 1 def _first_ipv4(packet: 'Packet') -> 'IP | None': @@ -478,18 +698,19 @@ def _first_ipv4(packet: 'Packet') -> 'IP | None': (:func:`pcapkit.toolkit.pcap.ipv4_reassembly`). ``packet['IP']`` finds the same layer but for one inside an ICMPv6 error's quote, which it reaches through the ``IPerror6`` there, or inside a layer the default - engine leaves raw, such as GRE (:issue:`1636`). + engine leaves raw, such as GRE (:issue:`1636`), or one past the frame's + layer bound (:issue:`1609`). """ from scapy.layers.inet import IP - for layer in _dissected(packet): - if isinstance(layer, IP): - return layer + for found in _dissected(packet): + if isinstance(found.layer, IP): + return found.layer return None -def _first_ipv6(packet: 'Packet') -> 'IPv6 | None': +def _first_ipv6(packet: 'Packet') -> 'Optional[_Layer]': """Fetch the first IPv6 layer of a Scapy packet, as the default engine dissects it. Args: @@ -498,53 +719,23 @@ def _first_ipv6(packet: 'Packet') -> 'IPv6 | None': Returns: The first :class:`~scapy.layers.inet6.IPv6` layer of the packet the default engine dissects too, however deep -- e.g. one carried in IPv4 - (:rfc:`4213`) -- or :data:`None` if there is none (see - :func:`_dissected`). + (:rfc:`4213`) -- with its place in the frame, or :data:`None` if there + is none (see :func:`_dissected`). Note: The default engine takes ``frame['IPv6']`` (:func:`pcapkit.toolkit.pcap.ipv6_reassembly`). ``packet['IPv6']`` finds the same layer but for one inside an ICMP or ICMPv6 error's quote, a later IPv4 fragment, or a layer the default engine leaves - raw, such as GRE (:issue:`1635`). + raw, such as GRE (:issue:`1635`), or one past the frame's layer bound + (:issue:`1609`). """ from scapy.layers.inet6 import IPv6 - for layer in _dissected(packet): - if isinstance(layer, IPv6): - return layer - return None - - -def _fragment_header(ipv6: 'IPv6') -> 'IPv6ExtHdrFragment | None': - """Fetch the Fragment header of a Scapy IPv6 layer's own extension header chain. - - Args: - ipv6: IPv6 layer of a dissected Scapy packet. - - Returns: - The first :class:`~scapy.layers.inet6.IPv6ExtHdrFragment` of the - extension headers that follow ``ipv6`` (see - :func:`_extension_headers`), or :data:`None` if there is none before the - upper-layer header. - - Note: - The default engine reads the Fragment header of that chain alone - (:func:`pcapkit.toolkit.pcap.ipv6_reassembly`). One behind the - upper-layer header, such as that of an IPv6 packet tunnelled in this - one, belongs to the packet inside (:rfc:`8200#section-4`, - :rfc:`2473#section-3.1`), and ``IPv6ExtHdrFragment in ipv6`` finds it - all the same (:issue:`1635`). - - """ - from scapy.layers.inet6 import IPv6ExtHdrFragment - - layer = ipv6.payload # type: Packet - while isinstance(layer, _extension_headers()): - if isinstance(layer, IPv6ExtHdrFragment): - return layer - layer = layer.payload + for found in _dissected(packet): + if isinstance(found.layer, IPv6): + return found return None @@ -553,7 +744,7 @@ def _tcp_behind_extension_headers(segment: 'bytes') -> 'tuple[bytes, bytes] | No Args: segment: Raw octets from the TCP header on, as far as the IPv4 layer - carrying it goes (see :func:`_tcp_carrier`). + carrying it goes (see :func:`_dissected`). Returns: The header, as many octets as the Data Offset counts, and the payload @@ -590,21 +781,15 @@ def _tcp_carrier(packet: 'Packet') -> 'tuple[IP | IPv6, TCP, bytes, bytes] | Non :class:`~scapy.layers.inet6.IPv6` layer above the packet's TCP layer, that layer, and the segment's header and payload (see :func:`_tcp_segment`); or :data:`None` if there is no such IP layer, or - the default engine reads no TCP there: it dissects nothing past a layer - above it (see :func:`_carries`) -- one it leaves raw, such as GRE + the default engine reads no TCP there: it dissects no TCP layer in the + frame (see :func:`_dissected`) -- one behind a link type it leaves raw + (:issue:`1650`), behind a layer it leaves raw, such as GRE (:issue:`1636`), an IPv4 layer whose header it rejects, or a later IPv4 - fragment (:issue:`1617`) -- or the TCP layer lies in the datagram an - ICMP or ICMPv6 error quotes (see :func:`_quoted`, :issue:`1627`). - - An IPv4 layer whose protocol names an AH or IPv6 extension header - carries what the default engine reads past those headers, by its own - rules and up to - :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` of - them in a row (see - :func:`~pcapkit.toolkit.pypcapfile._past_extension_headers`): a tunnel, - walked on as above, or a TCP segment (see - :func:`_tcp_behind_extension_headers`), each dissected from the - octets (:issue:`1606`). + fragment (:issue:`1617`), in the datagram an ICMP or ICMPv6 error quotes + (see :func:`_quoted`, :issue:`1627`), or past the frame's layer bound or + an extension header chain's (:issue:`1609`). A TCP segment the default + engine reads behind an AH or IPv6 extension header in IPv4 is split by + :func:`_tcp_behind_extension_headers` instead (:issue:`1606`). Note: The nearest is the innermost IP layer of an IP-in-IP tunnel @@ -612,49 +797,24 @@ def _tcp_carrier(packet: 'Packet') -> 'tuple[IP | IPv6, TCP, bytes, bytes] | Non endpoints', as for the default engine (:func:`pcapkit.toolkit.pcap.tcp_segment`). ``packet['IP']`` finds the outermost IPv4 layer instead, which is the tunnel's, or with IPv4 - carried in IPv6 the inner one (:issue:`1581`). The extension header - layers `Scapy`_ dissects between an IPv6 layer and its payload are not - IP layers, so are stepped over. - - Inside IPv4, `Scapy`_ dissects an AH alone of those headers, and behind - one only IPv6 and, for a Next Header of ``0``, IPv4, so the default - engine's rules are applied to the octets instead. It also dissects IPv6 - and AH inside a later fragment, binding neither to a zero fragment - offset as it binds TCP. + carried in IPv6 the inner one (:issue:`1581`). The extension headers + between an IPv6 layer and its payload are not IP layers, so are + stepped over. """ from scapy.layers.inet import IP, TCP from scapy.layers.inet6 import IPv6 - from scapy.packet import NoPayload - tcp = cast('TCP | None', packet.getlayer('TCP')) carrier = None # type: IP | IPv6 | None - layer = packet # type: Packet - while layer is not tcp: - if isinstance(layer, NoPayload) or not _carries(layer, root=layer is packet): - return None - if isinstance(layer, IP): - carrier = layer - if layer.proto in IPV4_EXTENSION_HEADERS: - upper = _past_extension_headers(layer.proto, _within_datagrams(layer.payload)) - if upper is not None and upper[0] == Enum_TransType.TCP: - segment = _tcp_behind_extension_headers(upper[1]) - return None if segment is None else (carrier, TCP(upper[1]), *segment) - if upper is None or upper[0] not in IP_TUNNELS: - return None - version, data = IP_TUNNELS[upper[0]], upper[1] - if len(data) < (IPV4_HEADER_LEN if version == 4 else IPV6_HEADER_LEN): - return None - layer = IP(data) if version == 4 else IPv6(data) - tcp = cast('TCP | None', layer.getlayer('TCP')) - continue - elif isinstance(layer, IPv6): - carrier = layer - layer = layer.payload - - if carrier is None or (segment := _tcp_segment(tcp)) is None: - return None - return carrier, tcp, *segment + for found in _dissected(packet): + if isinstance(found.layer, (IP, IPv6)): + carrier = found.layer + elif isinstance(found.layer, TCP): + if carrier is None: + return None + segment = _tcp_segment(found.layer) if found.segment is None else found.segment + return None if segment is None else (carrier, found.layer, *segment) + return None def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Address] | None': @@ -670,8 +830,9 @@ def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Ad * If the ``packet`` can be used for IPv4 reassembly. A packet can be reassembled if it contains IPv4 layer (:class:`scapy.layers.inet.IP`) -- the first of the frame the default engine dissects, so one carried in IPv6 counts but none an - ICMP or ICMPv6 error quotes or a layer that engine leaves raw carries, such as - GRE (see :func:`_first_ipv4`) -- whose header the default engine accepts (see + ICMP or ICMPv6 error quotes, a layer that engine leaves raw carries, such as + GRE, a link type it leaves raw carries, or one past the frame's layer bound (see + :func:`_first_ipv4`) -- whose header the default engine accepts (see :func:`_ipv4_accepted`) and lies inside every IPv6 packet carrying it, and the **DF** (:attr:`scapy.layers.inet.IP.flags.DF`) flag is :data:`False`. * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for IPv4 @@ -741,10 +902,11 @@ def ipv6_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv6Ad * If the ``packet`` can be used for IPv6 reassembly. A packet can be reassembled if it contains IPv6 layer (:class:`scapy.layers.inet6.IPv6`) -- the first of the frame the default engine dissects, so one carried in IPv4 counts but none an ICMP - or ICMPv6 error quotes, a later IPv4 fragment carries, or a layer that engine - leaves raw carries, such as GRE (see :func:`_first_ipv6`) -- and IPv6 Fragment - header (:rfc:`2460#section-4.5`, i.e., :class:`scapy.layers.inet6.IPv6ExtHdrFragment`) - in that layer's own extension header chain (see :func:`_fragment_header`). + or ICMPv6 error quotes, a later IPv4 fragment carries, a layer that engine leaves + raw carries, such as GRE, a link type it leaves raw carries, or one past the + frame's layer bound (see :func:`_first_ipv6`) -- and IPv6 Fragment header + (:rfc:`2460#section-4.5`) in that layer's own extension header chain, as far as + the default engine walks it (see :func:`_ipv6_chain`). * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for IPv6 reassembly (:term:`reasm.ipv6.packet`) will be returned; otherwise, returns :data:`None`. @@ -758,18 +920,23 @@ def ipv6_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv6Ad if scapy is None: raise ModuleNotFound("No module named 'scapy'", name='scapy') - if (ipv6 := _first_ipv6(packet)) is not None: - if (ipv6_frag := _fragment_header(ipv6)) is None: + if (found := _first_ipv6(packet)) is not None: + ipv6 = cast('IPv6', found.layer) + chain = _ipv6_chain(ipv6, found.depth) + if chain.fragment is None: return None # dismiss not fragmented packet - # NOTE: ``len()`` of a Scapy layer spans that layer and everything after - # it, so the difference is the unfragmentable part -- every octet before - # the Fragment header, which is the only part of the header the - # reassembled packet keeps (:rfc:`8200#section-4.5`). - hdr_len = len(ipv6) - len(ipv6_frag) + # NOTE: The unfragmentable part is every octet before the Fragment + # header, which is the only part of the header the reassembled packet + # keeps (:rfc:`8200#section-4.5`). The walk read the Fragment header off + # the octets, as `Scapy`_ dissects none behind a Mobility, HIP or Shim6 + # header (:issue:`1626`). + hdr_len = IPV6_HEADER_LEN + chain.fragment + ipv6_frag = chain.payload[chain.fragment:chain.fragment + 8] + field = int.from_bytes(ipv6_frag[2:4], 'big') # the fragment's data stops at the Payload Length, so neither it nor # ``tl`` below counts the frame's trailer (:issue:`1584`) - payload = bytearray(_within_datagrams(ipv6_frag.payload)) + payload = bytearray(chain.payload[chain.fragment + 8:]) data = IP_Packet( bufid=( @@ -777,22 +944,21 @@ def ipv6_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv6Ad ipaddress.ip_address(ipv6.src)), # source IP address cast('IPv6Address', ipaddress.ip_address(ipv6.dst)), # destination IP address - ipv6_frag.id, # identification - Enum_TransType.get(ipv6_frag.nh), # next header field in IPv6 Fragment Header + int.from_bytes(ipv6_frag[4:8], 'big'), # identification + Enum_TransType.get(ipv6_frag[0]), # next header field in IPv6 Fragment Header ), num=count, # original packet range number - # NOTE: Scapy reports ``IPv6ExtHdrFragment.offset`` in on-wire 8-octet - # units (:rfc:`8200#section-4.5`), but the reassembly machinery indexes + # NOTE: The Fragment Offset counts on-wire 8-octet units + # (:rfc:`8200#section-4.5`), but the reassembly machinery indexes # the datagram buffer with ``fo``, so it must be scaled into octets. - fo=ipv6_frag.offset * 8, # fragment offset + fo=(field >> 3) * 8, # fragment offset ihl=hdr_len, # header length, only headers before IPv6-Frag - mf=bool(ipv6_frag.m), # more fragment flag - # NOTE: ``len(ipv6)`` counts the Fragment header, so it overstates - # this by 8 -- and the reassembly machinery writes the payload over - # the span ``tl - ihl``, so using it would leave 8 octets of stray - # zeroes in every reassembled datagram. + mf=bool(field & 1), # more fragment flag + # NOTE: the Fragment header is not counted, as the reassembly + # machinery writes the payload over the span ``tl - ihl``, so counting + # it would leave 8 octets of stray zeroes in every reassembled datagram. tl=hdr_len + len(payload), # total length, header includes - header=bytes(ipv6)[:hdr_len], # raw bytes type header before IPv6-Frag + header=bytes(ipv6)[:IPV6_HEADER_LEN] + chain.payload[:chain.fragment], # header before IPv6-Frag payload=payload, # raw bytearray type payload after IPv6-Frag timestamp=float(packet.time), # capture timestamp ) @@ -814,9 +980,11 @@ def tcp_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'TCP_Packet | None if it contains TCP layer (:class:`scapy.layers.inet.TCP`) whose fixed header lies inside its datagram, and whose Data Offset counts at least that header, behind no IPv4 header the default engine rejects, in no later IPv4 fragment, in - no datagram an ICMP or ICMPv6 error quotes and in no layer the default engine - leaves raw, such as GRE, or a TCP segment the default engine reads behind an AH - or IPv6 extension header in IPv4. It is keyed by the IP layer that carries it -- + no datagram an ICMP or ICMPv6 error quotes, in no layer or link type the default + engine leaves raw, such as GRE or ``LINUX_SLL``, and inside the frame's layer + bound and an extension header chain's; or a TCP segment the default engine reads + behind an AH or IPv6 extension header in IPv4, or behind an IPv6 extension header + `Scapy`_ dissects nothing behind. It is keyed by the IP layer that carries it -- in an IP-in-IP tunnel, the innermost (see :func:`_tcp_carrier`). * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP reassembly (:term:`reasm.tcp.packet`) will be returned; otherwise, returns :data:`None`. diff --git a/tests/foundation/engines/test_ext_chain_limit_agreement_1604_unit.py b/tests/foundation/engines/test_ext_chain_limit_agreement_1604_unit.py index 60c51062d..d9b678026 100644 --- a/tests/foundation/engines/test_ext_chain_limit_agreement_1604_unit.py +++ b/tests/foundation/engines/test_ext_chain_limit_agreement_1604_unit.py @@ -14,11 +14,13 @@ over IPv6 out, so it is compared over IPv4 only, and ``declined`` checks that it warns of leaving out TCP over IPv6 exactly where ``default`` finds some. It imports on Python 3.11 only, and is skipped elsewhere. ``dpkt``'s adapters walk -the chain off the octets, and stop at the same depth too (#1609). +the chain off the octets, and stop at the same depth too. ``scapy`` walks the +chain over the octets by the default engine's rules, rather than through Scapy's +own layers, so stops there as well (#1609). -The other parsers walk the chain themselves, and cannot be told where to stop, -so where they find the TCP past the limit is recorded in :attr:`KNOWN_FAILURES` -under #1609 rather than hidden. Measured on scapy 2.7.0 and tshark 4.6.9. +The other parser, ``pyshark``, walks the chain through tshark, and cannot be +told where to stop, so where it finds the TCP past the limit is recorded in +:attr:`KNOWN_FAILURES` under #1609 rather than hidden. Measured on tshark 4.6.9. Every capture is built in a temporary directory, so no fixture is read. @@ -156,10 +158,6 @@ class TestExtensionHeaderChainAgreement(harness.RoundTripBase): STATUSES = ('ERROR', 'INPUTS', 'DECLINED') KNOWN_FAILURES = ( - Gap(1609, 'scapy dissects each extension header behind the last, with no limit of its own, ' - 'so finds the TCP past EXTENSION_HEADER_LIMIT; it gives up near 245 layers at the ' - 'default recursion limit, and so agrees at 1,000', - 'INPUTS', 'extra, default has none', ('ipv6-33/scapy/*',)), Gap(1609, 'tshark dissects up to about 496 extension headers, so finds the TCP past ' 'EXTENSION_HEADER_LIMIT, carried in IPv6; at 1,000 it gives up and agrees', 'INPUTS', 'extra, default has none', ('ipv6-33/pyshark/trace',)), diff --git a/tests/toolkit/test_scapy_ipv6_fragment_and_encap_walk_1635_1636_unit.py b/tests/toolkit/test_scapy_ipv6_fragment_and_encap_walk_1635_1636_unit.py index 829f0fd73..884d1f581 100644 --- a/tests/toolkit/test_scapy_ipv6_fragment_and_encap_walk_1635_1636_unit.py +++ b/tests/toolkit/test_scapy_ipv6_fragment_and_encap_walk_1635_1636_unit.py @@ -14,18 +14,16 @@ and its engine handed those to TCP reassembly and flow tracing, and to IPv4 and IPv6 reassembly (#1636). -The first layer is the one the link type selects. scapy takes it to carry -whatever `Scapy`_ dissects in it, and a layer behind it to carry IP only if it is -a VLAN tag, IP or an extension header. So at link types the default engine -leaves raw, where `Scapy`_ dissects two layers or more before IP -- HDLC-like -framing then PPP, 802.11 then LLC -- scapy reads no IP either. At those where it -dissects IP straight behind the first layer, such as LINUX_SLL, scapy still -reads what the default engine leaves raw, so no case here is one of those. - -Only scapy is compared with the default engine: dpkt gives no IPv6 reassembly -input for a 6in4 fragment (#1637). Behind more than 32 extension headers scapy -still reads what the default engine leaves raw, which #1609 tracks, so no case -here is that deep. +The first layer is the one the link type selects, and a layer behind it carries +IP only if it is a VLAN tag, IP or an extension header. So at link types the +default engine leaves raw, where `Scapy`_ dissects two layers or more before IP +-- HDLC-like framing then PPP, 802.11 then LLC -- scapy reads no IP either. Nor +does it at those where `Scapy`_ dissects IP straight behind the first layer, +such as LINUX_SLL, which +:mod:`tests.toolkit.test_scapy_linktype_and_layer_bounds_1609_1626_1650_unit` +covers with the frame and extension header bounds (#1650, #1609). + +Only scapy is compared with the default engine. Every capture is built here, so the module is in the unit tier. diff --git a/tests/toolkit/test_scapy_ipv6_payload_and_icmp_quote_1623_1627_unit.py b/tests/toolkit/test_scapy_ipv6_payload_and_icmp_quote_1623_1627_unit.py index 73bdad54c..3d1a87140 100644 --- a/tests/toolkit/test_scapy_ipv6_payload_and_icmp_quote_1623_1627_unit.py +++ b/tests/toolkit/test_scapy_ipv6_payload_and_icmp_quote_1623_1627_unit.py @@ -14,9 +14,7 @@ and that IPv4 to IPv4 reassembly. ``packet['IP']`` also reached the IPv4 inside the IPv6 packet a Redirected Header option holds, in a field rather than a payload. -dpkt is compared with the default engine as well. Behind more than 32 extension -headers, scapy still hands IPv4 reassembly the IPv4 the default engine leaves -raw, which #1609 tracks, so that case is left out here. +dpkt is compared with the default engine as well. Every capture is built here, so the module is in the unit tier. @@ -67,9 +65,8 @@ def redirect(packet: bytes) -> bytes: def ipv4_cases() -> 'dict[str, list[bytes]]': - """#1615/#1616's frames by case, less #1609's, then those of #1623 and the ICMP quotes.""" + """#1615/#1616's frames by case, then those of #1623 and the ICMP quotes.""" built = ipv4_input.cases() - del built['4in6 behind 33 extension headers'] # #1609 ethernet, ipv6 = ipv4_input.ethernet, ipv4_input.ipv6 def first(ident: int) -> bytes: diff --git a/tests/toolkit/test_scapy_linktype_and_layer_bounds_1609_1626_1650_unit.py b/tests/toolkit/test_scapy_linktype_and_layer_bounds_1609_1626_1650_unit.py new file mode 100644 index 000000000..e5194e85d --- /dev/null +++ b/tests/toolkit/test_scapy_linktype_and_layer_bounds_1609_1626_1650_unit.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- +"""scapy reads what the default engine dissects at every link type, and within its bounds. C.f. #1609, #1626, #1650. + +The default engine dissects the first layer of six link types alone, and keeps +the frames of any other raw. `Scapy`_ dissects IP and TCP straight behind the +headers of many of those, such as ``LINUX_SLL``, ``LINUX_SLL2``, ``PPP`` and the +``DLT_RAW`` of 12, and its engine handed them over. Behind a BSD loopback header +it read IPv6 at the ``AF_INET6`` of the host it ran on, and IPv4 at family 0 too, +where the default engine reads IPv4 at family 2 and IPv6 at 24, 28 and 30 on +every host (#1650). + +The default engine dissects no layer past the 64th of a frame, nor an +extension header past the 32nd in a row. `Scapy`_ stops at neither, and its +engine handed over the TCP, IPv4 and IPv6 fragments it found past them, behind +62 VLAN tags among them (#1609). + +`Scapy`_ dissects no TCP and no IPv4 behind an AH, and IPv4 where its Next Header +is 0; nothing behind a Mobility header; and neither HIP nor Shim6. The default +engine reads past each, so scapy now walks an IPv6 extension header chain over +the octets by the default engine's rules, as it walks those inside IPv4 (#1626). + +Every engine is compared frame for frame with the default engine, on every +input it hands TCP and IP reassembly and TCP flow tracing. Every capture is +built here, so the module is in the unit tier. + +.. _Scapy: https://scapy.net + +""" +from __future__ import annotations + +import importlib.util +import unittest +from typing import TYPE_CHECKING, NamedTuple +from unittest import mock + +from tests._support import reimport_once_per_class +from tests.foundation import _roundtrip as wire +from tests.toolkit import test_dpkt_ipv4_reassembly_input_1615_1616_unit as ipv4_input +from tests.toolkit.test_scapy_ipv6_fragment_and_encap_walk_1635_1636_unit import (C_TAG, fragment6, + read, tagged) +from tests.toolkit.test_tcp_carrier_walk_1606_1617_unit import ah, eth, options, tcp, v4, v6 + +if TYPE_CHECKING: + from typing import Any, Callable + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) +HAS_SCAPY = importlib.util.find_spec('scapy') is not None + +#: The inputs compared, as :func:`read` keys them. +ASPECTS = ('tcp', 'trace', 'ipv4', 'ipv6', 'ipv4 datagrams', 'ipv6 datagrams', 'ids', 'flows') + + +class Case(NamedTuple): + """Frames of one link type, and what the default engine reads in them.""" + + #: The capture's link type. + linktype: int + #: The frames. + frames: 'list[bytes]' + #: How many frames the default engine hands TCP, IPv4 and IPv6 reassembly. + reads: 'tuple[int, int, int]' + + +def version(packet: bytes) -> int: + return 6 if packet[0] >> 4 == 6 else 4 + + +def v4_fragments(ident: int) -> 'list[bytes]': + """The two fragments of a 24-octet IPv4 datagram.""" + return [ipv4_input.ipv4(b'A' * 16, ident=ident, mf=True), ipv4_input.ipv4(b'B' * 8, ident=ident, offset=16)] + + +def v6_fragments(ident: int, head: bytes = b'', code: int = 44) -> 'list[bytes]': + """The two fragments of a 24-octet IPv6 datagram, ``head`` (of code ``code``) before each Fragment header.""" + return [v6(head + fragment6(b'A' * 16, ident=ident, mf=True), code), + v6(head + fragment6(b'B' * 8, ident=ident, offset=16), code)] + + +def each(carry: 'Callable[[bytes], bytes]') -> 'list[bytes]': + """TCP in IPv4 and in IPv6, then two IPv4 and two IPv6 fragments, each carried by ``carry``.""" + return [carry(packet) for packet in (v4(tcp(1), 6), v6(tcp(2), 6), *v4_fragments(3), *v6_fragments(4))] + + +def ethertype(packet: bytes) -> bytes: + """The EtherType of the IP ``packet``, as octets.""" + return (0x86DD if version(packet) == 6 else 0x0800).to_bytes(2, 'big') + + +def sll(packet: bytes) -> bytes: + """A LINUX_SLL header (16 octets), then the IP ``packet``.""" + return b'\x00\x00\x00\x01\x00\x06' + wire.MAC_A + b'\x00\x00' + ethertype(packet) + packet + + +def sll2(packet: bytes) -> bytes: + """A LINUX_SLL2 header (20 octets), then the IP ``packet``.""" + return ethertype(packet) + b'\x00\x00\x00\x00\x00\x01\x00\x01\x00\x06' + wire.MAC_A + b'\x00\x00' + packet + + +def ppp(packet: bytes) -> bytes: + """A PPP frame with no HDLC-like framing, carrying the IP ``packet``.""" + return (0x0057 if version(packet) == 6 else 0x0021).to_bytes(2, 'big') + packet + + +def loopback(order: str, ipv4: int, ipv6: int) -> 'Callable[[bytes], bytes]': + """A BSD loopback header of family ``ipv4`` or ``ipv6``, written ``order``-endian, then the IP packet.""" + return lambda packet: (ipv6 if version(packet) == 6 else ipv4).to_bytes(4, order) + packet + + +def nest(packet: bytes, count: int, outer: int) -> bytes: + """``packet`` tunnelled in ``count`` IPv4 (``outer`` 4) or IPv6 (6) headers.""" + for _ in range(count): + proto = 41 if version(packet) == 6 else 4 + packet = v4(packet, proto) if outer == 4 else v6(packet, proto) + return packet + + +def vlan(packet: bytes, count: int) -> bytes: + """An Ethernet frame carrying the IP ``packet`` behind ``count`` 802.1Q tags.""" + return tagged(packet, *[C_TAG] * count) + + +def pairs(count: int, ipv6: bool) -> bytes: + """TCP behind ``count`` IPv4 packets, each with an AH, or IPv6 packets, each with a Hop-by-Hop header.""" + packet, nxt = tcp(1), 6 + for _ in range(count): + packet = v6(options(nxt) + packet, 0) if ipv6 else v4(ah(nxt) + packet, 51) + nxt = 41 if ipv6 else 4 + return packet + + +def chain(count: int, nxt: int, code: int = 60) -> bytes: + """``count`` 8-octet headers of type ``code``, each one PadN option, the last naming ``nxt``.""" + return b''.join(bytes([code if index < count - 1 else nxt, 0, 1, 4, 0, 0, 0, 0]) for index in range(count)) + + +def frag(nxt: int, mf: bool = False) -> bytes: + """A Fragment header of offset 0.""" + return bytes([nxt, 0, 0, int(mf)]) + (0xDEADBEEF).to_bytes(4, 'big') + + +#: The IPv6 extension headers of #1626, each a builder taking the Next Header it +#: names, and its own code. +HEADERS = { + 'fragment then ah': (lambda nxt: frag(51) + ah(nxt), 44), + 'ah': (ah, 51), + 'mobility': (lambda nxt: bytes([nxt, 0, 0, 0, 0, 0, 0, 0]), 135), + 'hip': (lambda nxt: bytes([nxt, 4, 1, 0x21, 0, 0, 0, 0]) + b'\x11' * 32, 139), + 'shim6': (lambda nxt: bytes([nxt, 0]) + bytes(6), 140), + 'hop-by-hop then mobility': (lambda nxt: options(135) + bytes([nxt, 0, 0, 0, 0, 0, 0, 0]), 0), + 'ah then hop-by-hop': (lambda nxt: ah(0) + options(nxt), 51), +} + + +def cases_1650() -> 'dict[str, Case]': + cases = { + 'linux_sll (113)': Case(113, each(sll), (0, 0, 0)), + 'linux_sll2 (276)': Case(276, each(sll2), (0, 0, 0)), + 'dlt_raw (12)': Case(12, each(lambda packet: packet), (0, 0, 0)), + 'ppp (9) without ff 03': Case(9, each(ppp), (0, 0, 0)), + # controls: the link types the default engine dissects + 'ethernet (1)': Case(1, each(eth), (2, 3, 2)), + 'raw (101)': Case(101, each(lambda packet: packet), (2, 3, 2)), + 'ipv4 (228)': Case(228, each(lambda packet: packet), (1, 3, 0)), + 'ipv6 (229)': Case(229, each(lambda packet: packet), (1, 0, 2)), + } + for linktype, name, order in ((0, 'null', 'little'), (0, 'null', 'big'), (108, 'loop', 'big')): + for family in (24, 28, 30): + cases[f'{name} ({linktype}) {order}-endian, ipv6 at {family}'] = Case( + linktype, each(loopback(order, 2, family)), (2, 3, 2)) + cases[f'{name} ({linktype}) {order}-endian, ipv6 at 10'] = Case( + linktype, each(loopback(order, 2, 10)), (1, 3, 0)) + cases[f'{name} ({linktype}) {order}-endian, family 0'] = Case( + linktype, each(loopback(order, 0, 0)), (0, 0, 0)) + cases['loop (108) little-endian'] = Case(108, each(loopback('little', 2, 24)), (0, 0, 0)) + cases['null (0) cut short'] = Case(0, [b'\x02\x00'], (0, 0, 0)) + cases['null (0) ipv4 cut short'] = Case(0, [loopback('little', 2, 24)(v4(tcp(1), 6))[:20]], (0, 0, 0)) + return cases + + +def cases_1609() -> 'dict[str, Case]': + # Ethernet is layer 1, and each VLAN tag, IP header and extension header one more + return { + 'tcp in 62 ipv4 (layer 64)': Case(1, [eth(nest(v4(tcp(1), 6), 61, 4))], (1, 1, 0)), + 'tcp in 63 ipv4 (layer 65)': Case(1, [eth(nest(v4(tcp(1), 6), 62, 4))], (0, 1, 0)), + 'tcp in 63 ipv6 (layer 65)': Case(1, [eth(nest(v6(tcp(1), 6), 62, 6))], (0, 0, 0)), + 'tcp behind 31 (ipv4, ah) pairs (layer 64)': Case(1, [eth(pairs(31, False))], (1, 1, 0)), + 'tcp behind 32 (ipv4, ah) pairs (layer 66)': Case(1, [eth(pairs(32, False))], (0, 1, 0)), + 'tcp behind 31 (ipv6, hop-by-hop) pairs (layer 64)': Case(1, [eth(pairs(31, True))], (1, 0, 0)), + 'tcp behind 32 (ipv6, hop-by-hop) pairs (layer 66)': Case(1, [eth(pairs(32, True))], (0, 0, 0)), + 'ipv4 fragments in 62 ipv6 (layer 64)': Case(1, [eth(nest(p, 62, 6)) for p in v4_fragments(1)], (0, 2, 0)), + 'ipv4 fragments in 63 ipv6 (layer 65)': Case(1, [eth(nest(p, 63, 6)) for p in v4_fragments(1)], (0, 0, 0)), + 'ipv6 fragments in 62 ipv4 (layer 65)': Case(1, [eth(nest(p, 62, 4)) for p in v6_fragments(1)], (0, 2, 0)), + 'tcp behind 61 vlan tags (layer 64)': Case(1, [vlan(v4(tcp(1), 6), 61)], (1, 1, 0)), + 'tcp behind 62 vlan tags (layer 65)': Case(1, [vlan(v4(tcp(1), 6), 62)], (0, 1, 0)), + 'ipv4 fragments behind 63 vlan tags (layer 65)': Case(1, [vlan(p, 63) for p in v4_fragments(1)], (0, 0, 0)), + 'ipv6 fragments behind 62 vlan tags (layer 65)': Case(1, [vlan(p, 62) for p in v6_fragments(1)], (0, 0, 0)), + 'tcp behind 32 extension headers': Case(1, [eth(v6(chain(32, 6) + tcp(1), 60))], (1, 0, 0)), + 'tcp behind 33 extension headers': Case(1, [eth(v6(chain(33, 6) + tcp(1), 60))], (0, 0, 0)), + 'tcp behind 33 hop-by-hop headers': Case(1, [eth(v6(chain(33, 6, 0) + tcp(1), 0))], (0, 0, 0)), + 'tcp behind 1000 extension headers': Case(1, [eth(v6(chain(1000, 6) + tcp(1), 60))], (0, 0, 0)), + 'fragment header after 32 extension headers': Case( + 1, [eth(p) for p in v6_fragments(1, chain(32, 44), 60)], (0, 0, 0)), + 'ipv4 fragments behind 33 extension headers': Case( + 1, [eth(v6(chain(33, 4) + p, 60)) for p in v4_fragments(1)], (0, 0, 0)), + # Ethernet, 50 IPv4 at 2 to 51, IPv6 at 52, then the chain: the frame bound cuts it + 'tcp behind 50 ipv4, ipv6 and 12 extension headers (layer 65)': Case( + 1, [eth(nest(v6(chain(12, 6) + tcp(1), 60), 50, 4))], (0, 1, 0)), + 'fragment header behind 50 ipv4, ipv6 and 12 extension headers (layer 65)': Case( + 1, [eth(nest(v6(chain(12, 44) + fragment6(b'A' * 16, ident=1, mf=True), 60), 50, 4))], (0, 1, 0)), + 'tcp behind 62 ipv4 and a hop-by-hop header (layer 65)': Case( + 1, [eth(nest(v4(options(6) + tcp(1), 0), 61, 4))], (0, 1, 0)), + 'tcp in 64 ipv4 at link type raw (layer 65)': Case(101, [nest(v4(tcp(1), 6), 63, 4)], (0, 1, 0)), + 'tcp in 63 ipv4 behind a loopback header (layer 65)': Case( + 0, [loopback('little', 2, 24)(nest(v4(tcp(1), 6), 62, 4))], (0, 1, 0)), + # controls, at the bound + 'tcp in 62 ipv6 (layer 64)': Case(1, [eth(nest(v6(tcp(1), 6), 61, 6))], (1, 0, 0)), + 'ipv6 fragments in 61 ipv4 (layer 64)': Case(1, [eth(nest(p, 61, 4)) for p in v6_fragments(1)], (0, 2, 2)), + 'ipv6 fragments behind 61 vlan tags (layer 64)': Case(1, [vlan(p, 61) for p in v6_fragments(1)], (0, 0, 2)), + 'fragment header after 31 extension headers': Case( + 1, [eth(p) for p in v6_fragments(1, chain(31, 44), 60)], (0, 0, 2)), + } + + +def cases_1626() -> 'dict[str, Case]': + cases = {} + for label, (head, code) in HEADERS.items(): + cases[f'ipv6, {label}, tcp'] = Case(1, [eth(v6(head(6) + tcp(1), code)), + eth(v6(head(6) + tcp(2, opts=b'\x02\x03\x05\xb4'), code))], + (2, 0, 2 if label == 'fragment then ah' else 0)) + cases[f'ipv6, {label}, ipv4 fragments'] = Case(1, [eth(v6(head(4) + p, code)) for p in v4_fragments(1)], + (0, 2, 2 if label == 'fragment then ah' else 0)) + cases[f'ipv6, {label}, 6in6'] = Case(1, [eth(v6(head(41) + v6(tcp(1), 6), code))], + (1, 0, 1 if label == 'fragment then ah' else 0)) + if label != 'fragment then ah': + cases[f'ipv6, {label}, then ipv6 fragments'] = Case(1, [eth(p) for p in v6_fragments(1, head(44), code)], + (0, 0, 2)) + # behind an extension header in IPv4, the IPv6 packet IPv6 reassembly takes + cases['6in4 fragments behind hop-by-hop in ipv4'] = Case(1, [eth(v4(options(41) + p, 0)) for p in v6_fragments(1)], + (0, 2, 2)) + cases['6in4 fragments behind mobility in ipv4'] = Case( + 1, [eth(v4(bytes([41, 0, 0, 0, 0, 0, 0, 0]) + p, 135)) for p in v6_fragments(1)], (0, 2, 2)) + # controls: what Scapy already dissects as the default engine does, and what neither reads past + cases['ipv6, fragment then destination options, tcp'] = Case(1, [eth(v6(frag(60) + options(6) + tcp(1), 44))], + (1, 0, 1)) + cases['ipv6, two fragment headers, tcp'] = Case(1, [eth(v6(frag(44, mf=True) + frag(6) + tcp(1), 44))], (1, 0, 1)) + cases['ipv6, mobility, tcp cut short'] = Case(1, [eth(v6(HEADERS['mobility'][0](6) + tcp(1)[:12], 135))], + (0, 0, 0)) + cases['ipv6, esp'] = Case(1, [eth(v6(b'\x00\x00\x01\x00\x00\x00\x00\x01' + bytes(8) + tcp(1), 50))], (0, 0, 0)) + cases['ipv6, 253'] = Case(1, [eth(v6(bytes([6, 0, 0, 0, 0, 0, 0, 0]) + tcp(1), 253))], (0, 0, 0)) + cases['ipv6, payload length cuts the chain'] = Case( + 1, [eth(v6(options(6) + tcp(1), 60)[:4] + (4).to_bytes(2, 'big') + v6(options(6) + tcp(1), 60)[6:])], (0, 0, 0)) + return cases + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +@unittest.skipUnless(HAS_SCAPY, 'scapy not installed') +class ScapyLinkTypeAndLayerBoundTests(unittest.TestCase): + """scapy hands on the TCP and IP the default engine dissects, frame for frame, and no other.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def agree(self, cases: 'dict[str, Case]') -> None: + reads, disagree = {}, [] + for label, case in cases.items(): + base = read(self, 'default', case.frames, case.linktype) + reads[label] = tuple(len(base[aspect]) for aspect in ('tcp', 'ipv4', 'ipv6')) + self.assertEqual(sorted(base['tcp']), sorted(base['trace']), label) + seen = read(self, 'scapy', case.frames, case.linktype) + if any(seen[aspect] != base[aspect] for aspect in ASPECTS): + disagree.append(label) + # what the default engine reads, as the cases say, then that scapy reads the same + self.assertEqual(reads, {label: case.reads for label, case in cases.items()}) + self.assertEqual(disagree, []) + + def test_no_ip_is_read_at_a_link_type_the_default_engine_leaves_raw(self) -> None: + # #1650 + self.agree(cases_1650()) + + def test_nothing_is_read_past_the_frame_or_extension_header_bound(self) -> None: + # #1609 + self.agree(cases_1609()) + + def test_tcp_and_ip_are_read_behind_every_extension_header_the_default_engine_reads_past(self) -> None: + # #1626 + self.agree(cases_1626()) + + def test_the_loopback_families_do_not_depend_on_the_host(self) -> None: + # #1650: Scapy binds IPv6 to the host's own AF_INET6 (10 here), and IPv4 + # to family 0 too; bound as on a BSD instead, it dissects otherwise, but + # scapy reads the same as the default engine either way + from scapy.layers.inet import IP + from scapy.layers.inet6 import IPv6 + from scapy.layers.l2 import Loopback + + bsd = [({'type': 2}, IP), ({'type': 24}, IPv6), ({'type': 28}, IPv6)] + for linktype, order in ((0, 'little'), (0, 'big'), (108, 'big')): + for ipv4, ipv6 in ((2, 10), (2, 24), (2, 28), (2, 30), (0, 0)): + frames = each(loopback(order, ipv4, ipv6)) + with self.subTest(linktype=linktype, order=order, families=(ipv4, ipv6)): + base = read(self, 'default', frames, linktype) + here = read(self, 'scapy', frames, linktype) + with mock.patch.object(Loopback, 'payload_guess', bsd): + there = read(self, 'scapy', frames, linktype) + self.assertEqual([(here[aspect], there[aspect]) for aspect in ASPECTS], + [(base[aspect], base[aspect]) for aspect in ASPECTS]) + + def test_a_packet_built_by_hand_is_read_by_its_first_layer(self) -> None: + # ``scapy.all`` binds every layer here, as the engine has it do (see pcapkit.toolkit.scapy) + import scapy.all # noqa: F401 # pylint: disable=unused-import + from scapy.layers.inet import IP, TCP + from scapy.layers.inet6 import IPv6 + from scapy.layers.l2 import CookedLinux, Dot3, Ether, Loopback, LoopbackOpenBSD + from scapy.packet import Raw + + from pcapkit.toolkit import scapy as toolkit + + segment = v4(tcp(1), 6) + + def reads(packet: 'Any') -> bool: + return toolkit.tcp_reassembly(packet) is not None + + self.assertEqual({ + 'ethernet': reads(Ether(eth(segment))), + 'ipv4': reads(IP(segment)), + 'ipv6': reads(IPv6(v6(tcp(1), 6))), + 'null, family 24': reads(Loopback(loopback('little', 2, 24)(v6(tcp(1), 6)))), + 'null, family 10': reads(Loopback(loopback('little', 2, 10)(v6(tcp(1), 6)))), + 'null, family 0': reads(Loopback(loopback('little', 0, 0)(segment))), + 'loop, family 2': reads(LoopbackOpenBSD(loopback('big', 2, 24)(segment))), + 'loop, little-endian': reads(LoopbackOpenBSD(loopback('little', 2, 24)(segment))), + 'linux_sll': reads(CookedLinux(sll(segment))), + 'tcp with no ip': reads(Ether() / TCP()), + '802.3': reads(Dot3() / Raw(segment)), + }, { + 'ethernet': True, 'ipv4': True, 'ipv6': True, 'null, family 24': True, 'null, family 10': False, + 'null, family 0': False, 'loop, family 2': True, 'loop, little-endian': False, 'linux_sll': False, + 'tcp with no ip': False, '802.3': False, + }) + # the link type the engine attaches stands over the first layer + packet = Ether(eth(segment)) + toolkit.attach_linktype(packet, 113) + self.assertEqual([toolkit.tcp_reassembly(packet), toolkit.ipv4_reassembly(packet)], [None, None]) + + def test_the_fragment_header_behind_a_header_scapy_reads_nothing_past_is_handed_over(self) -> None: + # #1626: the input is the octets either side of the Fragment header, as the default engine cuts them + import scapy.all # noqa: F401 # pylint: disable=unused-import + from scapy.layers.l2 import Ether + + from pcapkit.toolkit import scapy as toolkit + + mobility = bytes([44, 0, 0, 0, 0, 0, 0, 0]) + frame = eth(v6(mobility + fragment6(b'B' * 8, ident=7, nxt=6, offset=16), 135)) + b'\x00' * 6 + data = toolkit.ipv6_reassembly(Ether(frame), count=1) + assert data is not None + self.assertEqual((data.bufid[2:], data.fo, data.mf, data.ihl, data.tl, bytes(data.header), bytes(data.payload)), + ((7, 6), 16, False, 48, 56, frame[14:62], b'B' * 8)) + + +if __name__ == '__main__': + unittest.main()