From a176728e50ec2ba99dc43e18c2d4baafb2ae8026 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 14 Oct 2026 01:34:42 -0400 Subject: [PATCH] refactor(toolkit): revert pypcapfile's agreement-only code (#1708) Third-party engines are best effort (#1687), so the pypcapfile code that exists only to match the default engine comes out: - revert #1682, #1600, #1575 and #1535, and the pypcapfile hunks of #1655, #1608 and #1605: reading loopback and raw-IP link types, the frame layer bound, the VLAN and tunnel walks, keying tunnelled TCP by the innermost header, stepping over AH and extension headers, and checking options with the default engine's parsers - keep the helpers the dpkt and scapy adapters import, unchanged, until their own reverts drop them; their docstrings no longer say pypcapfile asks the default engine's parsers too - re-land #1576, #1577, #1590 and #1591 as guards in _network and _transport, #1513's once-per-capture warning for TCP straight over IPv6, and #1667's packet2dict fix: an undecoded frame is reported as its octets, not as hex text twice as long - keep #1569's record reader and #1690's packet2timestamp - delete the three test modules those reverts added, the pypcapfile timestamp agreement class, and #1682's link-type warning test; the re-landed guards' tests assert constants - drop what the reverts removed from the docs, READ_LINKTYPES included, and say that TCP behind a VLAN tag, in a tunnel or behind an extension header (an IPv6 one, or AH in IPv4) is now left out without a warning The pypcapfile modules, test_new_engine_parity_runtime.py, test_engine_runtime.py, the timestamp module and the project guards pass on 3.11 (pypcapfile 0.12.0) and 3.14. --- docs/source/contributing/pep.rst | 14 +- docs/source/ext.rst | 4 +- docs/source/index.rst | 6 +- .../pcapkit/foundation/engines/3rdparty.rst | 4 +- .../pcapkit/foundation/engines/index.rst | 12 +- .../pcapkit/protocols/internet/internet.rst | 7 +- docs/source/pcapkit/protocols/protocol.rst | 6 +- docs/source/pcapkit/toolkit/3rdparty.rst | 27 +- pcapkit/foundation/engines/pypcapfile.py | 51 +- pcapkit/toolkit/dpkt.py | 14 +- pcapkit/toolkit/pypcapfile.py | 1024 +++------------ pcapkit/toolkit/scapy.py | 10 +- tests/_dependency_gates.py | 2 +- .../engines/test_pypcapfile_engine.py | 42 +- ...pypcapfile_timestamp_rounding_1668_unit.py | 19 +- ...st_pypcapfile_ext_chain_limit_1604_unit.py | 149 --- .../test_pypcapfile_header_checks_unit.py | 264 ---- ...inktypes_and_layer_bound_1609_1667_unit.py | 345 ----- tests/toolkit/test_pypcapfile_unit.py | 1166 +++-------------- .../test_tunnelled_tcp_flow_key_1581_unit.py | 3 +- 20 files changed, 455 insertions(+), 2714 deletions(-) delete mode 100644 tests/toolkit/test_pypcapfile_ext_chain_limit_1604_unit.py delete mode 100644 tests/toolkit/test_pypcapfile_header_checks_unit.py delete mode 100644 tests/toolkit/test_pypcapfile_linktypes_and_layer_bound_1609_1667_unit.py diff --git a/docs/source/contributing/pep.rst b/docs/source/contributing/pep.rst index 64113121cd..9ca3b017a8 100644 --- a/docs/source/contributing/pep.rst +++ b/docs/source/contributing/pep.rst @@ -563,13 +563,15 @@ both of the following are worth knowing before reaching for them: sends the loader to ``find_library("pcap")``. No compiler and no :file:`pcap.h` are needed; a ``libpcap.so.1`` is. -Both engines support less than the ``default`` engine does, deliberately and -noisily: `pypcap`_ performs no protocol dissection, so it disables reassembly -*and* flow tracing; `pypcapfile`_ has no IPv6 decoder, so it disables IPv6 -reassembly while keeping IPv4 and TCP. Each gap is announced through an +Both engines support less than the ``default`` engine does, deliberately: +`pypcap`_ performs no protocol dissection, so it disables reassembly *and* flow +tracing; `pypcapfile`_ has no IPv6 decoder, so it disables IPv6 reassembly while +keeping IPv4 and TCP. Each gap is announced through an :class:`~pcapkit.utilities.warnings.AttributeWarning` or an outright exception -rather than by silently returning nothing -- -:doc:`/pcapkit/foundation/engines/index` tabulates them. +rather than by silently returning nothing, save that `pypcapfile`_ leaves TCP +behind a VLAN tag, in a tunnel or behind an extension header (an IPv6 one, or AH +in IPv4) out without one -- :doc:`/pcapkit/foundation/engines/index` tabulates +them. Adding a further engine does not mean adding handler methods to :class:`~pcapkit.foundation.extraction.Extractor`, as the thread describes. A new engine subclasses diff --git a/docs/source/ext.rst b/docs/source/ext.rst index 51b3e57526..534f3fb763 100644 --- a/docs/source/ext.rst +++ b/docs/source/ext.rst @@ -420,7 +420,9 @@ The available engines and the file formats they read: whatsoever, so they support neither reassembly nor flow tracing; `PyPCAPFile`_ has no IPv6 decoder, so it supports IPv4 and TCP reassembly but not IPv6. What matters is that the gap is *announced* -- each engine warns, or raises, for the - capability it cannot provide, rather than silently producing an empty result. See + capability it cannot provide, rather than silently producing an empty result, + save that `PyPCAPFile`_ leaves TCP behind a VLAN tag, in a tunnel or behind an + extension header (an IPv6 one, or AH in IPv4) out without a warning. See :doc:`pcapkit/foundation/engines/index` for the full table. .. note:: diff --git a/docs/source/index.rst b/docs/source/index.rst index 5729ada967..9971b4338c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -147,8 +147,10 @@ Speed is not free. Every third-party engine supports **less** than the dissection at all, so they offer neither reassembly nor flow tracing, and :class:`~pcapkit.foundation.engines.pypcapfile.PyPCAPFile` has no IPv6 decoder, so IPv6 reassembly is unavailable. Each gap is announced with a warning or an -exception rather than silently returning nothing; -:doc:`pcapkit/foundation/engines/index` tabulates them. +exception rather than silently returning nothing, save that +:class:`~pcapkit.foundation.engines.pypcapfile.PyPCAPFile` leaves TCP behind a +VLAN tag, in a tunnel or behind an extension header (an IPv6 one, or AH in IPv4) +out without one; :doc:`pcapkit/foundation/engines/index` tabulates them. Every engine also answers a preflight check -- :meth:`~pcapkit.foundation.engines.engine.EngineBase.unsupported_reason`, which diff --git a/docs/source/pcapkit/foundation/engines/3rdparty.rst b/docs/source/pcapkit/foundation/engines/3rdparty.rst index 95e6eee903..0eb686f10b 100644 --- a/docs/source/pcapkit/foundation/engines/3rdparty.rst +++ b/docs/source/pcapkit/foundation/engines/3rdparty.rst @@ -399,9 +399,7 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. TCP and UDP and nothing else. IPv6 reassembly is therefore unavailable and is disabled -- with an :class:`~pcapkit.utilities.warnings.AttributeWarning` -- when requested; IPv4 and TCP reassembly and TCP flow tracing remain - available. The IP packet of a BSD loopback or raw-IP frame, which it does - not decode, is read by :mod:`pcapkit.toolkit.pypcapfile` instead - (:issue:`1667`). PCAP-NG is rejected with a + available. PCAP-NG is rejected with a :exc:`~pcapkit.utilities.exceptions.FormatError`. `PyPCAPFile`_ 0.12.0 drops every record of a big-endian savefile, every record from a zero-length one on, and a truncated final record, so the engine reads the records itself, diff --git a/docs/source/pcapkit/foundation/engines/index.rst b/docs/source/pcapkit/foundation/engines/index.rst index c4a7bdcf89..65477ce44b 100644 --- a/docs/source/pcapkit/foundation/engines/index.rst +++ b/docs/source/pcapkit/foundation/engines/index.rst @@ -70,7 +70,8 @@ class hierarchy of :mod:`pcapkit.foundation.engines`: click PyPCAPFile "/pcapkit/foundation/engines/3rdparty.html#pcapkit.foundation.engines.pypcapfile.PyPCAPFile" Not every engine can do everything :class:`~pcapkit.foundation.extraction.Extractor` -offers, and the ones that cannot say so rather than quietly doing less: +offers, and the ones that cannot say so rather than quietly doing less, unless +their row says otherwise: +-----------------------------------------------------------------+---------------------------------------------------------------+ | Engine | Gap, and how it is surfaced | @@ -92,11 +93,12 @@ offers, and the ones that cannot say so rather than quietly doing less: | :class:`~pcapkit.foundation.engines.pypcapfile.PyPCAPFile` | no IPv6 decoder, hence no IPv6 reassembly -- disabled with an | | | :class:`~pcapkit.utilities.warnings.AttributeWarning`, and | | | :func:`~pcapkit.toolkit.pypcapfile.ipv6_reassembly` raises; | -| | TCP over IPv6, tunnelled in IP (e.g. 6in4) or not, is left | -| | out of TCP reassembly and flow tracing, with an | +| | TCP over IPv6 straight over Ethernet is left out of TCP | +| | reassembly and flow tracing, with an | | | :class:`~pcapkit.utilities.warnings.AttributeWarning` once | -| | per capture; IPv4 behind 802.1Q/802.1ad VLAN tags is read as | -| | it is untagged; PCAP savefiles only, otherwise | +| | per capture, and TCP behind a VLAN tag, in a tunnel or behind | +| | an extension header (an IPv6 one, or AH in IPv4) without one; | +| | PCAP savefiles only, otherwise | | | :exc:`~pcapkit.utilities.exceptions.FormatError` | +-----------------------------------------------------------------+---------------------------------------------------------------+ | :class:`~pcapkit.foundation.engines.dpkt.DPKT`, | no PCAP output, since each frame reaches the writer as a | diff --git a/docs/source/pcapkit/protocols/internet/internet.rst b/docs/source/pcapkit/protocols/internet/internet.rst index ff73019692..fde58f976d 100644 --- a/docs/source/pcapkit/protocols/internet/internet.rst +++ b/docs/source/pcapkit/protocols/internet/internet.rst @@ -37,8 +37,9 @@ A chain of IPv6 extension headers is dissected to at most a row, whether it follows IPv6 or another layer such as IPv4. What follows is kept as :class:`~pcapkit.protocols.misc.raw.Raw`, with one :exc:`~pcapkit.utilities.warnings.ProtocolWarning` (:issue:`1604`). The -``pypcapfile``, ``dpkt`` and ``scapy`` toolkits stop at the same depth. The -``pyshark`` engine leaves the walk to its own parser, which does not stop -there, so it may still find the upper layer past it (:issue:`1609`). +``dpkt`` and ``scapy`` toolkits stop at the same depth. The ``pyshark`` engine +leaves the walk to its own parser, which does not stop there, so it may still +find the upper layer past it (:issue:`1609`). The ``pypcapfile`` toolkit walks +no extension header chain, so it finds no upper layer behind one. .. autodata:: pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT diff --git a/docs/source/pcapkit/protocols/protocol.rst b/docs/source/pcapkit/protocols/protocol.rst index fe0f3b7b63..8448888702 100644 --- a/docs/source/pcapkit/protocols/protocol.rst +++ b/docs/source/pcapkit/protocols/protocol.rst @@ -116,9 +116,9 @@ there back out, as they do one the TCP parser rejected (:issue:`1518`). A protocol constructed from its own octets is not counted itself, so the count starts at the first layer it dissects. One chain of extension headers is also bounded on its own, by :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT`. -The ``pyshark`` engine dissects with its own parser, which does not apply this -bound (:issue:`1609`). The ``dpkt``, ``scapy`` and ``pypcapfile`` toolkits -count the layers they walk as the default engine does, and stop at it. +The ``pyshark`` and ``pypcapfile`` engines dissect with their own parsers, +which do not apply this bound (:issue:`1609`). The ``dpkt`` and ``scapy`` +toolkits count the layers they walk as the default engine does, and stop at it. .. autodata:: pcapkit.protocols.protocol.FRAME_LAYER_LIMIT diff --git a/docs/source/pcapkit/toolkit/3rdparty.rst b/docs/source/pcapkit/toolkit/3rdparty.rst index cdf164b77c..1cda0f236e 100644 --- a/docs/source/pcapkit/toolkit/3rdparty.rst +++ b/docs/source/pcapkit/toolkit/3rdparty.rst @@ -184,25 +184,15 @@ cannot be used for it. :data:`None` -- which would be indistinguishable from "this frame carries no IPv6 fragment". - Nor does it decode a VLAN tag or a tunnel. The adapters read an IPv4 packet - behind 802.1Q and 802.1ad tags as `PyPCAPFile`_ reads an untagged one, and - TCP tunnelled in IP (e.g. 4in4, 4in6) from its innermost IPv4 packet, keyed by - that packet's addresses as the default engine keys it, while TCP over IPv6, - tunnelled (e.g. 6in4, 6in6) or not, is left out of + TCP over IPv6 is therefore left out of :func:`~pcapkit.toolkit.pypcapfile.tcp_reassembly` and :func:`~pcapkit.toolkit.pypcapfile.tcp_traceflow`, with an - :class:`~pcapkit.utilities.warnings.AttributeWarning` once per capture. - - Nor does it decode a link layer but Ethernet. The adapters read the IP packet - of a BSD loopback (``NULL``, ``LOOP``) or raw-IP (``RAW``, ``IPV4``, - ``IPV6``) frame themselves, as the default engine dissects it (see - :data:`~pcapkit.toolkit.pypcapfile.READ_LINKTYPES`, :issue:`1667`). Every - other link type is left unread, as the default engine leaves it, and the - engine warns of it with an :class:`~pcapkit.utilities.warnings.AttributeWarning`. - Nor do the adapters read deeper than the default engine dissects: each VLAN - tag, IP header, AH and IPv6 extension header is one of the - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of a frame, and - nothing past the last of them is read (:issue:`1609`). + :class:`~pcapkit.utilities.warnings.AttributeWarning` once per capture when + the IPv6 packet is straight over Ethernet. `PyPCAPFile`_ decodes no link + layer but Ethernet, and no VLAN tag or tunnel, so a frame of any other link + type is left undecoded, with the engine's own warning, and TCP behind a VLAN + tag, in a tunnel or behind an extension header (an IPv6 one, or AH in IPv4) + is left out without one. .. autofunction:: pcapkit.toolkit.pypcapfile.ipv4_reassembly @@ -226,9 +216,6 @@ Auxiliary Functions Internal Definitions -------------------- -.. autodata:: pcapkit.toolkit.pypcapfile.READ_LINKTYPES - :no-value: - .. autodata:: pcapkit.toolkit.pypcapfile.TCP_MIN_HEADER_LEN .. autodata:: pcapkit.toolkit.pypcapfile.IPV4_FLAG_DF diff --git a/pcapkit/foundation/engines/pypcapfile.py b/pcapkit/foundation/engines/pypcapfile.py index 24808c608e..6f59fa5223 100644 --- a/pcapkit/foundation/engines/pypcapfile.py +++ b/pcapkit/foundation/engines/pypcapfile.py @@ -44,18 +44,15 @@ class PyPCAPFile(EngineBase['PCAPFilePacket']): * reads PCAP savefiles only, raising :exc:`~pcapkit.utilities.exceptions.FormatError` on PCAP-NG, * reads the savefile's records itself, into `PyPCAPFile`_'s own packet - objects, as `PyPCAPFile`_ 0.12.0 drops records the ``default`` engine - reads -- see :meth:`_read_records`, + objects, as `PyPCAPFile`_ 0.12.0 drops some of them -- see + :meth:`_read_records`, * disables IPv6 reassembly, warning as it does so, while leaving IPv4 and - TCP reassembly and TCP flow tracing in place, + TCP reassembly and TCP flow tracing in place, and * leaves TCP over IPv6 out of TCP reassembly and flow tracing, with one - :exc:`~pcapkit.utilities.warnings.AttributeWarning` per capture -- see - :mod:`pcapkit.toolkit.pypcapfile` -- and - * leaves the frames of a link type `PyPCAPFile`_ has no decoder for - undecoded, from which :mod:`pcapkit.toolkit.pypcapfile` reads the IP - packet of the BSD loopback and raw-IP ones itself, as the ``default`` - engine dissects it (see - :data:`~pcapkit.toolkit.pypcapfile.READ_LINKTYPES`). + :exc:`~pcapkit.utilities.warnings.AttributeWarning` per capture when the + IPv6 packet is straight over Ethernet, and TCP behind a VLAN tag, in a + tunnel or behind an extension header (an IPv6 one, or AH in IPv4) out + without one -- see :mod:`pcapkit.toolkit.pypcapfile`. The engine stops decoding at the network layer rather than descending into the transport layer. `PyPCAPFile`_ decoders *replace* the payload bytes of @@ -189,10 +186,8 @@ def run(self) -> 'None': support such operations. * if IPv6 reassembly is enabled, as :mod:`pcapfile` has no IPv6 decoder. - * if the savefile's link layer type is left unread: neither - :mod:`pcapfile` has a decoder for it nor - :mod:`pcapkit.toolkit.pypcapfile` reads it (see - :meth:`_get_decoder`). + * if :mod:`pcapfile` has no decoder for the savefile's link layer + type, in which case frames are left undecoded. Raises: FormatError: If the file format is not supported, i.e., not a PCAP @@ -309,36 +304,29 @@ def _get_decoder(self, linktype: 'int') -> 'Optional[Callable[..., Any]]': The decoder class, or :data:`None` when :mod:`pcapfile` has none. Warns: - AttributeWarning: If no decoder is available and the link layer type - is none of the :data:`~pcapkit.toolkit.pypcapfile.READ_LINKTYPES`, - as frames will then be left hexlified -- see :meth:`run` -- and - no reassembly or flow tracing is possible. The frames of those - link types are left undecoded too, but reassembly and flow - tracing read the IP packet in them, as the ``default`` engine - does, so nothing is lost there to warn of. + AttributeWarning: If no decoder is available, as frames will then be + left hexlified -- see :meth:`run` -- and no reassembly or flow + tracing is possible. """ - from pcapkit.toolkit.pypcapfile import READ_LINKTYPES # isort:skip - try: decoder = self._expkg.linklayer.clookup(linktype) except IndexError: # malformed entry in ``pcapfile.linklayer.__LL_TYPES__`` decoder = None if not callable(decoder): - if linktype not in READ_LINKTYPES: - warn(f'unrecognised link layer protocol: {self._dlink!r}; frames will be left ' - 'undecoded and all analysis functions ignored', AttributeWarning, - stacklevel=stacklevel()) + warn(f'unrecognised link layer protocol: {self._dlink!r}; frames will be left ' + 'undecoded and all analysis functions ignored', AttributeWarning, + stacklevel=stacklevel()) return None return decoder def _read_records(self, file: 'NamedStream', header: 'PCAPFileHeader') -> 'Iterator[PCAPFilePacket]': - """Read the savefile's records, as the ``default`` engine reads them. + """Read the savefile's records, every one of them. :func:`pcapfile.savefile._read_a_packet` (pypcapfile 0.12.0) drops - records the ``default`` engine reads, in two ways: + records in two ways: * It picks the byte order of each record header by testing ``hdrp[0].byteorder == 'big'`` -- :class:`bytes` against :class:`str`, @@ -354,9 +342,8 @@ def _read_records(self, file: 'NamedStream', builds with ``layers=0`` -- the frame hexlified, pointing at the savefile's own header -- so :meth:`_decode` and :mod:`pcapkit.toolkit.pypcapfile` see what :mod:`pcapfile` hands them. - It stops where :class:`~pcapkit.protocols.misc.pcap.frame.Frame` stops, - at a tail too short for a record header, and keeps a record whose capture - length runs past the data as that does: the capture length as declared, + It stops at a tail too short for a record header, and keeps a record + whose capture length runs past the data: the capture length as declared, the octets present as the frame. Args: diff --git a/pcapkit/toolkit/dpkt.py b/pcapkit/toolkit/dpkt.py index 23aba8236d..ada095f8db 100644 --- a/pcapkit/toolkit/dpkt.py +++ b/pcapkit/toolkit/dpkt.py @@ -649,9 +649,8 @@ def _ipv4_header_accepted(header: 'bytes') -> 'bool': Note: A header of version 4 with no options is not parsed, as the parser - checks nothing else of one; any other is asked of the parser itself, as - the pypcapfile engine asks it (see - :func:`pcapkit.toolkit.pypcapfile._default_accepts`). + checks nothing else of one; any other is asked of the parser itself + (see :func:`pcapkit.toolkit.pypcapfile._default_accepts`). """ return header[0] == 0x45 or _default_accepts('IPv4', bytes(header)) @@ -1023,11 +1022,10 @@ def _tcp_carrier(packet: 'Packet') -> 'Optional[_Segment]': (:func:`pcapkit.toolkit.pcap.tcp_segment`), as `DPKT`_ decodes one, and an IPv6 packet's extension headers do not come between, as it dissects them itself. Behind a header that is a layer of its own, it does not, so - the parser's answer stands there, as for the pypcapfile engine. The TCP - endpoints are the innermost IP layer's addresses, as for the default - engine; the outer ones are the tunnel's (:issue:`1581`). Each flag is - read from the flags octet alone, as the default engine reads it - (:issue:`1618`). + the parser's answer stands there. The TCP endpoints are the innermost IP + layer's addresses, as for the default engine; the outer ones are the + tunnel's (:issue:`1581`). Each flag is read from the flags octet alone, + as the default engine reads it (:issue:`1618`). .. _DPKT: https://dpkt.readthedocs.io diff --git a/pcapkit/toolkit/pypcapfile.py b/pcapkit/toolkit/pypcapfile.py index d6d3ba30a4..a972f95075 100644 --- a/pcapkit/toolkit/pypcapfile.py +++ b/pcapkit/toolkit/pypcapfile.py @@ -20,43 +20,16 @@ from "this frame carries no IPv6 fragment". For the same reason :func:`tcp_reassembly` and :func:`tcp_traceflow` see TCP - over IPv4 only, and decline a frame carrying TCP over IPv6 -- TCP whose - innermost IP layer is IPv6, tunnelled in IP (e.g. 6in4, 6in6) or not. That - loss is not silent: it is reported with one + over IPv4 only, and decline a frame carrying TCP over IPv6. When the IPv6 + packet is straight over Ethernet (below), that loss is reported with one :exc:`~pcapkit.utilities.warnings.AttributeWarning` per capture, on its first such frame, rather than one per frame. - `PyPCAPFile`_ decodes no VLAN tag either, so :func:`_network` reads past the - 802.1Q and 802.1ad tags the default engine steps over (see - :data:`VLAN_TAGS`), however deeply stacked, and decodes the IPv4 packet - behind them with `PyPCAPFile`_'s own decoder. TCP and IPv4 reassembly and TCP - flow tracing therefore see a tagged IPv4 frame as they see an untagged one. - Nor does it decode a tunnel, so :func:`_innermost` steps through the IPv4 - and IPv6 packets carried in IP (e.g. 4in4, 4in6) to the innermost, and - decodes it with the same decoder if it is IPv4: the TCP it carries is read - and keyed by its addresses, as the default engine keys it, not by the - tunnel's (:issue:`1581`). The warning follows the default engine's own rules - for where it finds TCP, read off the raw headers (see :func:`_upper_layer`). - IPv4 carried in IPv6 (4in6) also reaches :func:`ipv4_reassembly`, which the - default engine gives the first IPv4 layer of a frame (see - :func:`_ipv4_in_ipv6`), and TCP behind AH or an IPv6 extension header inside - IPv4 is read past them (see :func:`_transport`). - - `PyPCAPFile`_ decodes no link layer but Ethernet. The default engine also - dissects the IP packet of the BSD loopback and raw-IP link types, so the - adapters read it there themselves (see :data:`READ_LINKTYPES`, - :issue:`1667`), and leave the rest of the link types unread, as it does. - - Nor does it bound how deep it reads. The default engine dissects at most - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of a frame, - each VLAN tag, IP header, AH and IPv6 extension header one of them, so the - adapters read nothing past that either (see :func:`_walk_upper_layer`, - :issue:`1609`). - - `PyPCAPFile`_ checks neither that an IPv4 header was captured whole nor its - options, and the default engine rejects a header that fails either check, so - such a header is left out of reassembly and flow tracing alike (see - :func:`_ipv4_accepted`). + That warning covers an IPv6 packet straight over Ethernet whose Next Header + is TCP. `PyPCAPFile`_ decodes no VLAN tag (802.1Q), no QinQ (802.1ad) and no + tunnel (e.g. IPv6 in IPv4, or GRE), so TCP inside any of them, or behind an + extension header (an IPv6 one, or AH in IPv4), is left out without a + warning. Note also that `PyPCAPFile`_ decoders *replace* the payload bytes of the layer they decode. :class:`~pcapkit.foundation.engines.pypcapfile.PyPCAPFile` therefore @@ -79,28 +52,26 @@ from pcapkit.const.ipv6.extension_header import ExtensionHeader as Enum_ExtensionHeader from pcapkit.const.reg.ethertype import EtherType as Enum_EtherType -from pcapkit.const.reg.linktype import LinkType as Enum_LinkType from pcapkit.const.reg.transtype import TransType as Enum_TransType from pcapkit.foundation.reassembly.data.ip import Packet as IP_Packet from pcapkit.foundation.reassembly.data.tcp import Packet as TCP_Packet from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet from pcapkit.protocols.internet.internet import EXTENSION_HEADER_LIMIT from pcapkit.protocols.internet.ipv4 import IPv4 as Protocol_IPv4 -from pcapkit.protocols.link.loopback import family_byteorder -from pcapkit.protocols.protocol import FRAME_LAYER_LIMIT -from pcapkit.protocols.schema.internet.ipv6 import jumbo_payload_length from pcapkit.protocols.transport.tcp import TCP as Protocol_TCP from pcapkit.utilities.exceptions import ProtocolError, UnsupportedCall, stacklevel from pcapkit.utilities.warnings import AttributeWarning, warn if TYPE_CHECKING: - from collections import OrderedDict, deque + from collections import OrderedDict from ipaddress import IPv4Address, IPv6Address from typing import Any, Optional from pcapfile.protocols.network.ip import IP from pcapfile.structs import pcap_packet as Packet + from pcapkit.const.reg.linktype import LinkType as Enum_LinkType + __all__ = [ 'packet2timestamp', 'ipv4_header', 'packet2chain', 'packet2dict', 'ipv4_reassembly', 'ipv6_reassembly', 'tcp_reassembly', 'tcp_traceflow', @@ -126,30 +97,38 @@ #: Length of the fixed IPv6 header, :rfc:`8200#section-3`. IPV6_HEADER_LEN = 40 +#: How many captures :func:`_decline_ipv6_tcp` remembers having warned about. +#: Each one pins a packet of its own in memory (see :func:`_capture_of`), so the +#: memory is bounded and the oldest is forgotten first. Up to this many captures +#: read at the same time warn exactly once each. +IPV6_TCP_WARNED_LIMIT = 32 + +#: The captures :func:`_decline_ipv6_tcp` has warned about, oldest first. Each +#: key :func:`_capture_of` gives maps to the anchor that keeps that key unique. +_ipv6_tcp_warned = collections.OrderedDict() # type: OrderedDict[int, Any] + +# NOTE: the definitions from here to ``packet2timestamp``, and +# ``_default_accepts``, ``_default_extension_header`` and +# ``_walk_extension_headers`` below, are kept for :mod:`pcapkit.toolkit.dpkt` +# and :mod:`pcapkit.toolkit.scapy`: they import the rest, and +# ``_walk_extension_headers`` uses ``AH_HEADER_LEN`` and +# ``_default_extension_header``. This module's adapters use none of them +# (#1708). + #: Length of the fixed part of an IP Authentication Header (AH), before its -#: Integrity Check Value, :rfc:`4302#section-2`. A Payload Length of ``0`` -#: declares a shorter header, which the default engine rejects. +#: Integrity Check Value, :rfc:`4302#section-2`. AH_HEADER_LEN = 12 -#: IP protocol numbers of the headers :func:`_past_extension_headers` reads -#: past inside IPv4, on to the protocol each one's Next Header field names, as -#: the default engine does: it dispatches IPv4's protocol through the registry -#: IPv6 uses (:attr:`Internet.__proto__ -#: `), and each of these -#: parsers then decodes what its Next Header names. That is AH and the IPv6 -#: Hop-by-Hop Options, Routing, Fragment, Destination Options, Mobility and HIP -#: headers. ESP is left out, as its Next Header is encrypted, and Shim6, whose -#: parser refuses IPv4. +#: IP protocol numbers of the headers :func:`_walk_extension_headers` steps +#: over inside IPv4: AH and the IPv6 Hop-by-Hop Options, Routing, Fragment, +#: Destination Options, Mobility and HIP headers. IPV4_EXTENSION_HEADERS = frozenset(( Enum_TransType.HOPOPT, Enum_TransType.IPv6_Route, Enum_TransType.IPv6_Frag, Enum_TransType.AH, Enum_TransType.IPv6_Opts, Enum_TransType.Mobility_Header, Enum_TransType.HIP, )) -#: EtherTypes of the VLAN tags :func:`_network` reads past: the 802.1Q customer -#: tag and the 802.1ad service tag, which QinQ stacks. These are the two the -#: default engine dissects (see :attr:`Link.__proto__ -#: `). +#: EtherTypes of the 802.1Q customer tag and the 802.1ad service tag. VLAN_TAGS = frozenset(( Enum_EtherType.Customer_VLAN_Tag_Type, Enum_EtherType.IEEE_Std_802_1Q_Service_VLAN_tag_identifier, @@ -159,74 +138,20 @@ #: then the EtherType of what follows the tag. VLAN_TAG_LEN = 4 -#: Link types `PyPCAPFile`_ has no decoder for, whose IP packet the default -#: engine dissects all the same (see :attr:`Frame.__proto__ -#: `), and which the -#: adapters therefore read themselves (see :func:`_link_network`, -#: :issue:`1667`): BSD loopback, ``NULL`` and ``LOOP``, and raw IP, ``RAW``, -#: ``IPV4`` and ``IPV6``. Every other link type but ``ETHERNET``, which -#: `PyPCAPFile`_ decodes, is left unread, as the default engine leaves it raw. -READ_LINKTYPES = frozenset(( - Enum_LinkType.NULL, Enum_LinkType.LOOP, Enum_LinkType.RAW, - Enum_LinkType.IPV4, Enum_LinkType.IPV6, -)) - -#: Length of a BSD loopback header: the address family. -LOOPBACK_HEADER_LEN = 4 - -#: Address families of the IP packets the default engine dissects behind a BSD -#: loopback header, each mapped to its IP version (see :attr:`Loopback.__proto__ -#: `): ``AF_INET``, and the -#: ``AF_INET6`` of each of the BSDs. -LOOPBACK_IP_VERSIONS = {2: 4, 24: 6, 28: 6, 30: 6} # type: dict[int, int] - -#: IP protocol numbers of the IP-in-IP tunnels the default engine dissects -#: (see :attr:`Internet.__proto__ -#: `), each mapped to the -#: IP version it carries: IPv4 (protocol 4, :rfc:`2003`) and IPv6 (protocol 41, +#: IP protocol numbers of the IP-in-IP tunnels, each mapped to the IP version +#: it carries: IPv4 (protocol 4, :rfc:`2003`) and IPv6 (protocol 41, #: :rfc:`2473` and :rfc:`4213`). IP_TUNNELS = {Enum_TransType.IPv4: 4, Enum_TransType.IPv6: 6} -#: IPv6 extension headers that :func:`_upper_layer` steps over on its way -#: to the upper-layer protocol, as the default engine does: the ones +#: IPv6 extension headers that precede the upper-layer protocol: the ones #: :rfc:`8200#section-4.1` orders before it, less ESP, whose payload is -#: encrypted, plus the Mobility, HIP and Shim6 headers. Those three are sized in -#: 8-octet units like most of the rest, and the default engine dissects whatever -#: protocol their next header field names. The experimental 253 and 254 are left -#: out, because the default engine dissects nothing after them. +#: encrypted, plus the Mobility, HIP and Shim6 headers. IPV6_EXTENSION_HEADERS = frozenset(( Enum_ExtensionHeader.HOPOPT, Enum_ExtensionHeader.IPv6_Opts, Enum_ExtensionHeader.IPv6_Route, Enum_ExtensionHeader.IPv6_Frag, Enum_ExtensionHeader.AH, Enum_ExtensionHeader.Mobility_Header, Enum_ExtensionHeader.HIP, Enum_ExtensionHeader.Shim6, )) -#: The warning :func:`_decline_tcp` gives for each kind of TCP that -#: :func:`_left_out` finds this engine cannot read. TCP tunnelled in IP is no -#: kind of its own: the innermost IP layer carrying it is read if it is IPv4, -#: and is TCP over IPv6 otherwise (see :func:`_innermost`, :issue:`1581`). -TCP_LEFT_OUT = { - 'ipv6': "TCP over IPv6 is left out of TCP reassembly and flow tracing, as 'pypcapfile' " - 'has no IPv6 decoder', -} - -#: How many warnings :func:`_decline_tcp` remembers having given, each for one -#: kind in :data:`TCP_LEFT_OUT` and one capture. Each one pins a packet of its -#: own in memory (see :func:`_capture_of`), so the memory is bounded and the -#: oldest is forgotten first. Up to this many read at the same time warn exactly -#: once each. -TCP_WARNED_LIMIT = 32 - -#: The warnings :func:`_decline_tcp` has given, oldest first. Each is keyed by -#: the key :func:`_capture_of` gives and the kind, and maps to the anchor that -#: keeps that key unique. -_tcp_warned = collections.OrderedDict() # type: OrderedDict[tuple[int, str], Any] - -#: The frame :func:`_network` last warned it could not decode: the key -#: :func:`_capture_of` gives, the packet index and the anchor that keeps the key -#: unique. The adapters each fetch the network layer of a frame in turn, so this -#: is what holds them to one warning per frame between them. -_undecoded_last = collections.deque(maxlen=1) # type: deque[tuple[int, int, Any]] - def packet2timestamp(packet: 'Packet') -> 'float': """Calculate the timestamp of a PyPCAPFile packet. @@ -236,7 +161,7 @@ def packet2timestamp(packet: 'Packet') -> 'float': Returns: Timestamp of the packet, in seconds since the epoch: the :obj:`float` - nearest the exact value, as the default engine hands over. + nearest the exact value. Note: `PyPCAPFile`_ keeps the two halves of the per-packet timestamp apart, and @@ -423,340 +348,197 @@ def _is_raw(layer: 'Any') -> 'bool': return isinstance(layer, (bytes, bytearray, memoryview)) -def _untag(link: 'Any') -> 'Optional[tuple[int, bytes, int]]': - """Read past the VLAN tags of an Ethernet frame whose payload is undecoded. +def _network(packet: 'Packet') -> 'Optional[IP]': + """Fetch the decoded IPv4 layer of a PyPCAPFile packet, if any. Args: - link: Decoded PyPCAPFile link layer. + packet: PyPCAPFile packet. Returns: - The EtherType after the last of the tags in :data:`VLAN_TAGS`, the raw - octets it heads, and their place in the frame (c.f. - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`), the Ethernet - header being the first layer and each tag one more -- for an untagged - frame, the frame's own EtherType and payload, and ``2``. :data:`None` if - `PyPCAPFile`_ decoded the payload, a tag is cut short, or what follows - the tags is past that bound, where the default engine dissects nothing - (:issue:`1609`). + The decoded :class:`~pcapfile.protocols.network.ip.IP` layer, or + :data:`None` when the frame was not decoded that far -- which is the + case for every non-IPv4 frame, `PyPCAPFile`_ having no other network + layer decoder -- or its header length (IHL) runs past the captured + octets (:issue:`1591`). Note: - `PyPCAPFile`_ leaves a payload it has no decoder for hex-encoded (see - :func:`_maybe_unhex`), so this reads the raw octets and decodes nothing. + `PyPCAPFile`_ slices the header at its IHL however few octets there + are, so a header cut short has fewer options than its IHL declares. .. _PyPCAPFile: https://github.com/kisom/pypcapfile """ - ethertype = getattr(link, 'type', None) # type: Optional[int] - payload = getattr(link, 'payload', None) # type: Any - if ethertype is None or not _is_raw(payload): + link = packet.packet + if _is_raw(link): return None - data, depth = _maybe_unhex(bytes(payload)), 2 - while ethertype in VLAN_TAGS: - if depth == FRAME_LAYER_LIMIT or len(data) < VLAN_TAG_LEN: - return None - ethertype, data, depth = int.from_bytes(data[2:4], 'big'), data[VLAN_TAG_LEN:], depth + 1 - return ethertype, data, depth - - -@functools.lru_cache(maxsize=2) -def _default_accepts(name: 'str', header: 'bytes') -> 'bool': - """Test if the default engine's parser accepts a header. - - Args: - name: ``'IPv4'`` or ``'TCP'``, the parser to ask. - header: The whole header, options included, and nothing after it. - - Returns: - :data:`False` if the parser raises anything at all, as the default - engine then leaves the layer undissected (see - :func:`~pcapkit.utilities.decorators.beholder`); else :data:`True`. - - Note: - This is for the options, whose rules are the parser's and too many for - a header walk to follow, so it is asked only of a header that has some. - It is given the header alone and stops at its own layer, so nothing - past the header is dissected. What the parser logs and warns of, a - rejected header's :exc:`~pcapkit.utilities.exceptions.ProtocolError` - included, it logs and warns of as it does when the default engine reads - the same frame; neither the warnings filters nor the logger's level are - changed. The answer depends on the octets alone, so the last two are - remembered: the adapters each fetch a frame's layers in turn, and this - parses a frame's IPv4 and TCP headers once each between them. - - """ - protocol = Protocol_IPv4 if name == 'IPv4' else Protocol_TCP - try: - protocol(header, len(header), protocol=name) - except Exception: # as beholder does - return False - return True + payload = getattr(link, 'payload', None) # type: Any + if type(payload).__name__ != 'IP': + return None + if payload.hl * 4 > IPV4_HEADER_LEN and ( + IPV4_HEADER_LEN + len(_maybe_unhex(bytes(payload.opt))) < payload.hl * 4): + return None + return payload -def _ipv4_accepted(ipv4: 'IP') -> 'bool': - """Test if the default engine reads a PyPCAPFile IPv4 packet as an IPv4 layer. +def _ipv6_carries_tcp(packet: 'Packet') -> 'bool': + """Test if a PyPCAPFile frame is an IPv6 packet carrying TCP. Args: - ipv4: PyPCAPFile IPv4 packet. + packet: PyPCAPFile packet. Returns: - :data:`False` if its header length (IHL) runs past the captured octets - (:issue:`1591`), or its options are ones the default engine's IPv4 - parser rejects (:issue:`1596`); else :data:`True`. + :data:`True` if the frame is IPv6 straight over Ethernet and its Next + Header is TCP; otherwise :data:`False`. Note: - `PyPCAPFile`_ slices the header at its IHL however few octets there - are, and never checks the options. A header with none is not parsed: - all the default engine checks of one is that it is version 4, with an - IHL of at least 5 and 20 octets captured, and `PyPCAPFile`_'s decoder - refuses any other. + `PyPCAPFile`_ leaves an IPv6 frame's payload undecoded and hex-encoded + (see :func:`_maybe_unhex`), so this reads the Next Header field of the + raw IPv6 header and decodes nothing. .. _PyPCAPFile: https://github.com/kisom/pypcapfile """ - if ipv4.hl * 4 == IPV4_HEADER_LEN: - return True - options = _maybe_unhex(bytes(ipv4.opt)) - if IPV4_HEADER_LEN + len(options) < ipv4.hl * 4: + link = packet.packet + if _is_raw(link) or getattr(link, 'type', None) != Enum_EtherType.Internet_Protocol_version_6: return False - return _default_accepts('IPv4', ipv4_header(ipv4)) - - -def _link_network(packet: 'Packet') -> 'Optional[tuple[int, bytes, int]]': - """Find the IP packet the default engine dissects in a frame `PyPCAPFile`_ has no decoder for. - - Args: - packet: PyPCAPFile packet, whose frame `PyPCAPFile`_ left undecoded. - - Returns: - The packet's IP version, its octets on to the end of the frame, and its - place in the frame (c.f. - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`), by the - capture's link type, if it is one of the :data:`READ_LINKTYPES`: ``2`` - behind a BSD loopback header whose address family is one of the - :data:`LOOPBACK_IP_VERSIONS`, read in the byte order - :func:`~pcapkit.protocols.link.loopback.family_byteorder` gives -- - ``LOOP``'s in network byte order, ``NULL``'s in the order its octets - show; and ``1`` for an ``IPV4`` or ``IPV6`` frame, or a ``RAW`` one of - the IP version its first octet gives, 4 or 6. :data:`None` for any other - link type, family or version, a frame cut short of its loopback header, - and an empty one, where the default engine dissects no IP either. - - Note: - The frame is hex-encoded, as the engine reads it (see - :meth:`PyPCAPFile._read_records - `), and - the link type is the savefile header's, which the packet points at. - - .. _PyPCAPFile: https://github.com/kisom/pypcapfile - - """ - linktype = packet.header[0].ll_type - if linktype not in READ_LINKTYPES: - return None - octets = _maybe_unhex(bytes(packet.packet)) - if linktype in (Enum_LinkType.NULL, Enum_LinkType.LOOP): - if len(octets) < LOOPBACK_HEADER_LEN: - return None - family = octets[:LOOPBACK_HEADER_LEN] - version = LOOPBACK_IP_VERSIONS.get(int.from_bytes(family, family_byteorder(family, linktype))) - return None if version is None else (version, octets[LOOPBACK_HEADER_LEN:], 2) - if linktype == Enum_LinkType.RAW: - version = octets[0] >> 4 if octets else 0 - if version not in (4, 6): - return None - else: - version = 4 if linktype == Enum_LinkType.IPV4 else 6 - return version, octets, 1 - - -def _network(packet: 'Packet', count: 'int' = -1) -> 'Optional[IP]': - """Fetch the outermost IPv4 layer of a PyPCAPFile packet, if any. - - Args: - packet: PyPCAPFile packet. - count: Packet index, for the warning :func:`_network_layer` gives. - Returns: - The IPv4 layer :func:`_network_layer` fetches, or :data:`None`. + payload = getattr(link, 'payload', None) # type: Any + if not _is_raw(payload): + return False - """ - layer = _network_layer(packet, count) - return None if layer is None else layer[0] + data = _maybe_unhex(bytes(payload)) + return len(data) >= IPV6_HEADER_LEN and data[6] == Enum_TransType.TCP -def _network_layer(packet: 'Packet', count: 'int' = -1) -> 'Optional[tuple[IP, int]]': - """Fetch the outermost IPv4 layer of a PyPCAPFile packet, and its place in the frame. +def _capture_of(packet: 'Packet') -> 'tuple[int, Any]': + """Identify the capture a PyPCAPFile packet was read from. Args: packet: PyPCAPFile packet. - count: Packet index, for the warning below. Returns: - The decoded :class:`~pcapfile.protocols.network.ip.IP` layer and its - place in the frame (c.f. - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`), or :data:`None` - when the frame carries no IPv4 right after its link layer -- - `PyPCAPFile`_ having no other network layer decoder -- or the default - engine rejects its header (see :func:`_ipv4_accepted`). Behind an - Ethernet header that is ``2``, and one more for each VLAN tag (see - :func:`_untag`). A frame of a link type `PyPCAPFile`_ has no decoder for - is read by :func:`_link_network`, and its IPv4 packet decoded by - :func:`_decode_ipv4`. - - Warns: - AttributeWarning: If an IPv4 packet behind VLAN tags will not decode, - in the words, and with the category, - :meth:`PyPCAPFile._decode ` - uses for an untagged one -- once per frame, however many adapters - fetch its network layer. + A key, equal for every packet of one capture, and an anchor: while the + anchor is alive, no packet of another capture has the same key. Note: - `PyPCAPFile`_ decodes no VLAN tag, so it leaves an IPv4 packet behind - one undecoded. That packet is decoded here, past every tag that - :func:`_untag` reads past, with the same - :class:`~pcapfile.protocols.network.ip.IP` decoder and the same depth - :class:`~pcapfile.protocols.linklayer.ethernet.Ethernet` uses for an - untagged frame, so the two decode alike; one that will not decode is - left out with a warning, as an untagged one is. The engine warns of an - untagged frame as it reads it, and this of a tagged one as reassembly or - flow tracing does. Over the other link types the engine decodes nothing, - so has nothing to warn of. - - .. _PyPCAPFile: https://github.com/kisom/pypcapfile + :attr:`pcap_packet.header ` is a + :mod:`ctypes` pointer to the savefile header. Each read of the field + builds a new pointer object, but every one of them points at the same + header, so the header's address is the key. The pointer object read + here is the anchor: it keeps its packet alive and, through that packet, + the header itself, so no other capture's header can be allocated at + that address while it is held. A header that is not a :mod:`ctypes` + pointer is keyed by its identity instead, and anchors itself. """ - link = packet.packet - if _is_raw(link): - found = _link_network(packet) - if found is None or found[0] != 4: - return None - decoded = _decode_ipv4(found[1]) - return None if decoded is None else (decoded, found[2]) - - payload = getattr(link, 'payload', None) - if type(payload).__name__ == 'IP': - return (payload, 2) if _ipv4_accepted(payload) else None - if getattr(link, 'type', None) not in VLAN_TAGS: - return None - - untagged = _untag(link) - if untagged is None or untagged[0] != Enum_EtherType.Internet_Protocol_version_4: - return None - - # NOTE: imported only once there is something to decode, as in - # :func:`tcp_reassembly`; ``layers=0`` stops at the network layer, as the - # engine does (see :attr:`PyPCAPFile.LAYERS - # `). - from pcapfile.protocols.network.ip import IP # isort:skip - + anchor = packet.header try: - ipv4 = IP(untagged[1], 0) - except (struct.error, AssertionError, ValueError, IndexError, KeyError) as error: - key, anchor = _capture_of(packet) - if not _undecoded_last or _undecoded_last[0][:2] != (key, count): - _undecoded_last.append((key, count, anchor)) - warn(f'Frame {count}: {Enum_LinkType.ETHERNET!r} decoding failed ({error!r}); ' - 'frame left undecoded', AttributeWarning, stacklevel=stacklevel()) - return None - return (ipv4, untagged[2]) if _ipv4_accepted(ipv4) else None + return ctypes.addressof(anchor.contents), anchor + except (AttributeError, TypeError, ValueError): + return id(anchor), anchor -def _outer_ipv6(packet: 'Packet') -> 'Optional[tuple[bytes, int]]': - """Fetch the outermost IPv6 packet of a PyPCAPFile frame, which `PyPCAPFile`_ cannot decode. +def _decline_ipv6_tcp(packet: 'Packet', count: 'int') -> 'None': + """Warn that a frame carrying TCP over IPv6 is left out, once per capture. Args: - packet: PyPCAPFile packet. - - Returns: - The octets of the IPv6 packet right after the frame's link layer, on to - the end of the frame, and its place in the frame: behind an Ethernet - header and any VLAN tags (see :func:`_untag`), or in a frame of a link - type `PyPCAPFile`_ has no decoder for (see :func:`_link_network`). - :data:`None` if there is none. + packet: PyPCAPFile packet, which :func:`_network` found no IPv4 layer in. + count: Packet index. - .. _PyPCAPFile: https://github.com/kisom/pypcapfile + Warns: + AttributeWarning: If ``packet`` carries TCP over IPv6 (see + :func:`_ipv6_carries_tcp`) and is the first such frame of its + capture to reach :func:`tcp_reassembly` or :func:`tcp_traceflow`. + Later frames of the same capture are left out without another + warning, and every other capture warns once of its own, however + their frames interleave. Only the :data:`IPV6_TCP_WARNED_LIMIT` + captures warned about most recently are remembered, so with more + than that read at once a forgotten one warns again: the warning can + repeat, but is never omitted. """ - link = packet.packet - if _is_raw(link): - found = _link_network(packet) - return None if found is None or found[0] != 6 else (found[1], found[2]) - untagged = _untag(link) - if untagged is None or untagged[0] != Enum_EtherType.Internet_Protocol_version_6: - return None - return untagged[1], untagged[2] + key, anchor = _capture_of(packet) + if key in _ipv6_tcp_warned: + return + if not _ipv6_carries_tcp(packet): + return + _ipv6_tcp_warned[key] = anchor + while len(_ipv6_tcp_warned) > IPV6_TCP_WARNED_LIMIT: + _ipv6_tcp_warned.popitem(last=False) + warn(f"Frame {count}: TCP over IPv6 is left out of TCP reassembly and flow tracing, as " + "'pypcapfile' has no IPv6 decoder; later such frames of this capture are left out " + 'without another warning', AttributeWarning, stacklevel=stacklevel()) -def _ipv4_in_ipv6(packet: 'Packet') -> 'Optional[IP]': - """Fetch the IPv4 layer of a PyPCAPFile frame carrying IPv4 in IPv6 (4in6). + +def _transport(ipv4: 'IP') -> 'Optional[bytes]': + """Fetch the verbatim TCP segment carried by a PyPCAPFile IPv4 packet. Args: - packet: PyPCAPFile packet. + ipv4: PyPCAPFile IPv4 packet. Returns: - The first IPv4 packet inside an IPv6 packet -- directly, behind - extension headers, or inside more IPv6 (6in6) -- decoded as - :func:`_network` decodes one behind VLAN tags, or :data:`None` if there - is none, it will not decode, or the default engine rejects its header - (see :func:`_ipv4_accepted`). The outer IPv6 packet is the one - :func:`_outer_ipv6` fetches, and each IPv6 header is stepped over as - :func:`_walk_upper_layer` does, so no IPv4 packet past - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the - frame is found (:issue:`1609`). + The raw TCP segment, up to the packet's Total Length, or :data:`None` + if the packet does not carry a TCP header. A later fragment carries + none: from its offset on, its data is a slice of the datagram's payload + (:rfc:`791#section-3.2`, :issue:`1576`). Nor does a segment too short + for a header, or whose Data Offset is below 5 words (:issue:`1590`). Note: - `PyPCAPFile`_ has no IPv6 decoder, so leaves the IPv4 inside undecoded. - This is for :func:`ipv4_reassembly` alone, which the default engine - gives the first IPv4 layer of a frame however deep; TCP is keyed by the - innermost instead (see :func:`_innermost`). Nor does an IPv4 packet that - will not decode warn: `PyPCAPFile`_ never decodes IPv6, so has nothing - to warn of. + `PyPCAPFile`_ keeps every octet after the header as the payload, so a + frame padded to the Ethernet minimum carries its padding there. The + segment is cut at the Total Length to leave it out (:issue:`1577`). A + Total Length of ``0``, which TCP segmentation offload leaves in a + capture taken on the sending host, keeps the rest of the frame + (:issue:`1547`). .. _PyPCAPFile: https://github.com/kisom/pypcapfile """ - outer = _outer_ipv6(packet) - if outer is None: + if ipv4.p != Enum_TransType.TCP or ipv4.off: + return None + + segment = ipv4.payload + if not _is_raw(segment): return None - upper = _walk_upper_layer(6, *outer) - while upper is not None and upper[0] == Enum_TransType.IPv6: - upper = _walk_upper_layer(6, upper[1], upper[2]) - if upper is None or upper[0] != Enum_TransType.IPv4: + segment = _maybe_unhex(bytes(segment)) + if ipv4.len: + segment = segment[:max(ipv4.len - ipv4.hl * 4, 0)] + if len(segment) < TCP_MIN_HEADER_LEN or segment[12] >> 4 < 5: # Data Offset, in words return None - return _decode_ipv4(upper[1]) + return segment -def _decode_ipv4(data: 'bytes') -> 'Optional[IP]': - """Decode an IPv4 packet that `PyPCAPFile`_ left undecoded. +@functools.lru_cache(maxsize=2) +def _default_accepts(name: 'str', header: 'bytes') -> 'bool': + """Test if pcapkit's own IPv4 or TCP parser accepts a header. Args: - data: Raw octets from the IPv4 header on, as far as the IP packet - carrying it goes (see :func:`_upper_layer`), or the frame of a link - type `PyPCAPFile`_ has no decoder for (see :func:`_link_network`). + name: ``'IPv4'`` or ``'TCP'``, the parser to ask. + header: The whole header, options included, and nothing after it. Returns: - The packet, decoded as :func:`_network` decodes one behind VLAN tags, or - :data:`None` if it will not decode or the default engine rejects its - header (see :func:`_ipv4_accepted`), which then dissects nothing past it. + :data:`False` if the parser raises anything at all; else :data:`True`. - .. _PyPCAPFile: https://github.com/kisom/pypcapfile + Note: + The parser stops at its own layer, so nothing past the header is + dissected. The answer depends on the octets alone, so the last two are + remembered. """ - # NOTE: imported only once there is something to decode, as in :func:`_network`. - from pcapfile.protocols.network.ip import IP # isort:skip - + protocol = Protocol_IPv4 if name == 'IPv4' else Protocol_TCP try: - ipv4 = IP(data, 0) - except (struct.error, AssertionError, ValueError, IndexError, KeyError): - return None - return ipv4 if _ipv4_accepted(ipv4) else None + protocol(header, len(header), protocol=name) + except Exception: # as beholder does + return False + return True @functools.lru_cache(maxsize=8) def _default_extension_header(protocol: 'int', data: 'bytes') -> 'Optional[tuple[int, int]]': - """Read an extension header inside IPv4 with the default engine's own parser. + """Read an extension header inside IPv4 with pcapkit's own parser. Args: protocol: The header's protocol number, one of @@ -764,18 +546,9 @@ def _default_extension_header(protocol: 'int', data: 'bytes') -> 'Optional[tuple data: Raw octets from the header on, as far as the IPv4 packet goes. Returns: - The header's Next Header field and its length, or :data:`None` if the - default engine leaves it undissected: its parser raises anything at - all (see :func:`~pcapkit.utilities.decorators.beholder`), or fewer - octets were captured than the header needs. - - Note: - The parser is the one the default engine dispatches ``protocol`` to - inside IPv4, called as it calls it, so its options, routing data, - message or parameters are checked as the default engine checks them. - It stops at its own layer, so nothing past the header is dissected, and - the last few answers are remembered, as in :func:`_default_accepts`, - which this logs and warns as. + The header's Next Header field and its length, or :data:`None` if its + parser raises anything at all, or fewer octets were captured than the + header needs. """ # pylint: disable-next=protected-access @@ -791,60 +564,27 @@ def _default_extension_header(protocol: 'int', data: 'bytes') -> 'Optional[tuple return int(nxt), header.length -def _past_extension_headers(protocol: 'int', data: 'bytes') -> 'Optional[tuple[int, bytes]]': - """Step over the headers at the head of an IPv4 payload that the default engine reads past. +def _walk_extension_headers(protocol: 'int', data: 'bytes') -> 'Optional[tuple[int, bytes, int]]': + """Step over the headers in :data:`IPV4_EXTENSION_HEADERS` at the head of an IPv4 payload. Args: protocol: The IPv4 packet's protocol number. data: Raw octets of its payload. Returns: - The protocol number after the last of the :data:`IPV4_EXTENSION_HEADERS` - and the octets from its header on -- ``protocol`` and ``data`` - themselves if ``protocol`` is none of them -- or :data:`None` if the - default engine leaves one of them undissected. An AH is - ``(Payload Length + 2) * 4`` octets long (:rfc:`4302#section-2.2`), and - no shorter than :data:`AH_HEADER_LEN`; any other is read by - :func:`_default_extension_header` (:issue:`1597`). Past + The protocol number after the last of them, the octets from its header + on, and how many headers were stepped over -- ``protocol``, ``data`` and + ``0`` if ``protocol`` is none of them -- or :data:`None` if one of them + is cut short or rejected, or more than :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` - of them in a row, the default engine leaves the rest undissected too - (:issue:`1604`). - - Note: - Inside IPv4, the default engine decodes what an IPv6 Fragment header - names whatever its offset, so a later fragment is read past too. Over - IPv6, :func:`_upper_layer` steps over these as the extension headers - they are there. - - """ - walked = _walk_extension_headers(protocol, data) - return None if walked is None else walked[:2] - - -def _walk_extension_headers(protocol: 'int', data: 'bytes', - limit: 'int' = EXTENSION_HEADER_LIMIT) -> 'Optional[tuple[int, bytes, int]]': - """Step over the headers :func:`_past_extension_headers` steps over, and count them. - - Args: - protocol: The IPv4 packet's protocol number. - data: Raw octets of its payload. - limit: How many of them in a row the default engine dissects at most, - by default - :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT`; - fewer where the frame has fewer layers left (see - :func:`_walk_ipv4_payload`). - - Returns: - What :func:`_past_extension_headers` returns, then how many headers it - stepped over, each of them one layer of the frame to the default engine - (c.f. :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`); or - :data:`None` where it returns :data:`None`, or a header past ``limit`` - follows, which the default engine leaves undissected with the rest. + follow in a row. An AH is ``(Payload Length + 2) * 4`` octets long + (:rfc:`4302#section-2.2`), and no shorter than :data:`AH_HEADER_LEN`; + any other is read by :func:`_default_extension_header`. """ depth = 0 while protocol in IPV4_EXTENSION_HEADERS: - if depth >= limit: + if depth == EXTENSION_HEADER_LIMIT: return None depth += 1 if protocol == Enum_TransType.AH: @@ -860,366 +600,6 @@ def _walk_extension_headers(protocol: 'int', data: 'bytes', return protocol, data, depth -def _upper_layer(version: 'int', data: 'bytes') -> 'Optional[tuple[int, bytes]]': - """Step over an IP header to the upper-layer protocol it carries. - - Args: - version: IP version of the header ``data`` starts with, ``4`` or ``6``. - data: Raw octets of the IP packet. - - Returns: - The upper-layer protocol number and the octets from its header on, or - :data:`None` if no upper-layer header follows. - - An IPv4 header must be version 4 with a header length of at least - :data:`IPV4_HEADER_LEN` octets, all of them captured, and what follows - it ends at its Total Length, as in :func:`_ipv4_payload`, and is - followed past the headers :func:`_past_extension_headers` steps over. - An IPv6 packet ends at its Payload Length or, where that is ``0``, at the Jumbo Payload - Length a jumbogram's Hop-by-Hop Options header gives (:rfc:`2675`) -- - lacking one, it has no payload -- and is followed past the extension headers in - :data:`IPV6_EXTENSION_HEADERS`; a first fragment is read past its - Fragment header as if that header were not there. A later fragment of - either has no upper-layer header, since from its offset on it carries a - slice of the payload (:rfc:`791#section-3.2`, :rfc:`8200#section-4.5`). - Nor does a truncated header or header chain, or a chain of more than - :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` - extension headers (:issue:`1604`). - - Note: - Each of these rules is the default engine's own, as are the ones - :func:`_left_out` applies to the TCP header, so that :func:`_innermost` - stops at the IP layer the default engine finds TCP in, and the warning - :func:`_decline_tcp` gives fires on the frames it finds TCP over IPv6 in. - This reads only as far as the upper-layer protocol number and decodes - nothing, so it does not see an IPv6 extension header option the default - engine rejects, which leaves the header raw. Every IPv4 header on the - way is checked by :func:`_innermost` (see :func:`_ipv4_accepted`), and a - TCP header the default engine's TCP parser rejects directly over its IP - layer, tunnelled or not, it still reads out of the raw octets left - behind (:func:`pcapkit.toolkit.pcap.tcp_segment`). - - The header is read as the first layer of its frame, where - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` leaves room for - more headers than either chain bound allows, so that bound never applies - here; :func:`_walk_upper_layer` reads one deeper in a frame. - - """ - walked = _walk_upper_layer(version, data, 1) - return None if walked is None else walked[:2] - - -def _walk_upper_layer(version: 'int', data: 'bytes', depth: 'int') -> 'Optional[tuple[int, bytes, int]]': - """Step over an IP header to the upper-layer protocol it carries, and place that in the frame. - - Args: - version: IP version of the header ``data`` starts with, ``4`` or ``6``. - data: Raw octets of the IP packet. - depth: The IP header's place in its frame, the first layer being ``1`` - (c.f. :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`), at - most that bound. - - Returns: - What :func:`_upper_layer` returns, by its rules, then the upper layer's - place in the frame: one past the IP header and each header stepped over - after it, every one of them a layer of its own to the default engine. Or - :data:`None` where :func:`_upper_layer` returns :data:`None`, or the - upper layer is past - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`, where the default - engine dissects nothing (:issue:`1609`); no header past it is read. - - """ - room = FRAME_LAYER_LIMIT - depth # layers the frame has left after the IP header - if version == 4: - if len(data) < IPV4_HEADER_LEN or data[0] >> 4 != 4: - return None - ihl = (data[0] & 0x0F) * 4 - if ihl < IPV4_HEADER_LEN or len(data) < ihl: - return None - if struct.unpack_from('!H', data, 6)[0] & 0x1FFF: # a later fragment - return None - total = struct.unpack_from('!H', data, 2)[0] - return _walk_ipv4_payload(data[9], data[ihl:max(total, ihl)] if total else data[ihl:], depth) - - if len(data) < IPV6_HEADER_LEN: - return None - extent = struct.unpack_from('!H', data, 4)[0] - if not extent: - extent = jumbo_payload_length(data[6], data[IPV6_HEADER_LEN:]) or 0 - data = data[:IPV6_HEADER_LEN + extent] - nxt, offset, headers = data[6], IPV6_HEADER_LEN, 0 - bound = min(EXTENSION_HEADER_LIMIT, room) - while nxt in IPV6_EXTENSION_HEADERS: - if headers >= bound or len(data) < offset + 8: - return None - headers += 1 - if nxt == Enum_ExtensionHeader.IPv6_Frag: # fixed length, second octet reserved - if struct.unpack_from('!H', data, offset + 2)[0] >> 3: # a later fragment - return None - length = 8 - elif nxt == Enum_ExtensionHeader.AH: # length in 4-octet units, less two - length = (data[offset + 1] + 2) * 4 - else: # length in 8-octet units, less one - length = (data[offset + 1] + 1) * 8 - nxt, offset = data[offset], offset + length - if len(data) < offset or headers >= room: - return None - return nxt, data[offset:], depth + headers + 1 - - -def _walk_ipv4_payload(protocol: 'int', data: 'bytes', depth: 'int') -> 'Optional[tuple[int, bytes, int]]': - """Step over the headers at the head of an IPv4 payload, and place what follows in the frame. - - Args: - protocol: The IPv4 packet's protocol number. - data: Raw octets of its payload. - depth: The IPv4 header's place in its frame, as for - :func:`_walk_upper_layer`. - - Returns: - What :func:`_past_extension_headers` returns, then the place in the - frame of the layer it names: one past the IPv4 header and each header - stepped over. :data:`None` where :func:`_past_extension_headers` returns - :data:`None`, or that layer or a header before it is past - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` (:issue:`1609`). - - """ - room = FRAME_LAYER_LIMIT - depth # layers the frame has left after the IPv4 header - walked = _walk_extension_headers(protocol, data, min(EXTENSION_HEADER_LIMIT, room)) - if walked is None or walked[2] >= room: - return None - return walked[0], walked[1], depth + walked[2] + 1 - - -def _innermost(packet: 'Packet', count: 'int' = -1) -> 'tuple[Optional[IP], Optional[tuple[int, bytes]], int]': - """Fetch the innermost IP layer of a PyPCAPFile frame, the one carrying its TCP. - - Args: - packet: PyPCAPFile packet. - count: Packet index, for the warning :func:`_network_layer` gives. - - Returns: - The innermost IPv4 packet and :data:`None`, if the innermost IP layer is - IPv4; :data:`None` and the upper-layer protocol and octets of the - innermost IPv6 packet, as :func:`_walk_upper_layer` gives them, if it is - IPv6; otherwise neither. Then the innermost IP layer's place in the - frame (c.f. :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT`), for - :func:`_transport`; ``0`` if there is none. - - The outermost IP layer is the IPv4 packet :func:`_network_layer` - fetches, or the IPv6 packet :func:`_outer_ipv6` does. Each IPv4 or IPv6 - packet one of them carries in the :data:`IP_TUNNELS`, directly or behind - the headers :func:`_walk_upper_layer` steps over, is the next, as deep as - they go and no deeper than the frame's layer bound (:issue:`1609`). An - IPv4 packet in a tunnel is decoded by :func:`_decode_ipv4`, and one it - does not decode ends the walk with neither, as the default engine then - dissects no TCP past it. - - Note: - The default engine keys a TCP segment by the IP layer nearest above it - (:func:`pcapkit.toolkit.pcap.tcp_segment`), which in a tunnel is the - innermost, so the TCP endpoints are its addresses and not the tunnel's - (:issue:`1581`). - - """ - ipv4 = None # type: Optional[IP] - layer = _network_layer(packet, count) - if layer is not None: - outer4, depth = layer - # a later fragment has no upper-layer header, see ``_upper_layer`` - if (outer4.off or (outer4.p not in IP_TUNNELS and outer4.p not in IPV4_EXTENSION_HEADERS) - or not _is_raw(outer4.payload)): - return outer4, None, depth - ipv4, upper = outer4, _walk_ipv4_payload(outer4.p, _ipv4_payload(outer4), depth) - else: - outer6 = _outer_ipv6(packet) - if outer6 is None: - return None, None, 0 - depth, upper = outer6[1], _walk_upper_layer(6, *outer6) - - while upper is not None and upper[0] in IP_TUNNELS: - version, data, depth = IP_TUNNELS[upper[0]], upper[1], upper[2] - upper = _walk_upper_layer(version, data, depth) - if version == 6: - ipv4 = None - elif (ipv4 := _decode_ipv4(data)) is None: - return None, None, depth - if ipv4 is not None: - return ipv4, None, depth - return None, None if upper is None else upper[:2], depth - - -def _left_out(upper: 'Optional[tuple[int, bytes]]') -> 'Optional[str]': - """Name the TCP a PyPCAPFile frame carries that this engine cannot read. - - Args: - upper: The upper-layer protocol and octets of the frame's innermost IP - layer, if that is IPv6, as :func:`_innermost` gives them. - - Returns: - ``'ipv6'`` if they are a TCP header, i.e. the frame carries TCP over - IPv6, directly or behind extension headers, tunnelled in IP or not; - otherwise :data:`None`. An ESP payload cannot be seen into, so does not - count. - - The TCP header has to be at least :data:`TCP_MIN_HEADER_LEN` octets - long, as its Data Offset gives it, with that many captured. The rest of - the header need not be: the default engine's TCP parser rejects one cut - short, e.g. by the snapshot length, and then reads the segment out of - the raw octets left behind (:func:`pcapkit.toolkit.pcap.tcp_segment`, - :issue:`1518`). - - """ - if upper is None or upper[0] != Enum_TransType.TCP: - return None - segment = upper[1] - if len(segment) < TCP_MIN_HEADER_LEN: - return None - if (segment[12] >> 4) * 4 < TCP_MIN_HEADER_LEN: # the Data Offset, in octets - return None - return 'ipv6' - - -def _capture_of(packet: 'Packet') -> 'tuple[int, Any]': - """Identify the capture a PyPCAPFile packet was read from. - - Args: - packet: PyPCAPFile packet. - - Returns: - A key, equal for every packet of one capture, and an anchor: while the - anchor is alive, no packet of another capture has the same key. - - Note: - :attr:`pcap_packet.header ` is a - :mod:`ctypes` pointer to the savefile header. Each read of the field - builds a new pointer object, but every one of them points at the same - header, so the header's address is the key. The pointer object read - here is the anchor: it keeps its packet alive and, through that packet, - the header itself, so no other capture's header can be allocated at - that address while it is held. A header that is not a :mod:`ctypes` - pointer is keyed by its identity instead, and anchors itself. - - """ - anchor = packet.header - try: - return ctypes.addressof(anchor.contents), anchor - except (AttributeError, TypeError, ValueError): - return id(anchor), anchor - - -def _decline_tcp(packet: 'Packet', upper: 'Optional[tuple[int, bytes]]', count: 'int') -> 'None': - """Warn that a frame carrying TCP this engine cannot read is left out. - - Args: - packet: PyPCAPFile packet, which :func:`_transport` found no TCP segment in. - upper: The upper-layer protocol and octets of its innermost IP layer, if - that is IPv6, as :func:`_innermost` gives them. - count: Packet index. - - Warns: - AttributeWarning: If ``packet`` carries TCP that :func:`_left_out` names, - and is the first frame of its capture carrying that kind of TCP to - reach :func:`tcp_reassembly` or :func:`tcp_traceflow`. Later frames - of the same capture and kind are left out without another warning, - and every other capture or kind warns once of its own, however their - frames interleave. Only the :data:`TCP_WARNED_LIMIT` warnings given - most recently are remembered, so with more than that read at once a - forgotten one is given again: a warning can repeat, but is never - omitted. - - """ - key, anchor = _capture_of(packet) - if all((key, kind) in _tcp_warned for kind in TCP_LEFT_OUT): - return - kind = _left_out(upper) - if kind is None or (key, kind) in _tcp_warned: - return - - _tcp_warned[key, kind] = anchor - while len(_tcp_warned) > TCP_WARNED_LIMIT: - _tcp_warned.popitem(last=False) - warn(f'Frame {count}: {TCP_LEFT_OUT[kind]}; later such frames of this capture are left out ' - 'without another warning', AttributeWarning, stacklevel=stacklevel()) - - -def _ipv4_payload(ipv4: 'IP') -> 'bytes': - """Fetch the raw payload of a PyPCAPFile IPv4 packet, as far as its Total Length. - - Args: - ipv4: PyPCAPFile IPv4 packet, whose payload `PyPCAPFile`_ left undecoded. - - Returns: - The octets after the header, up to the Total Length. A Total Length of - ``0``, which TCP segmentation offload leaves in a capture taken on the - sending host, leaves the rest of the frame; one shorter than the header - leaves none (:issue:`1547`). - - Note: - `PyPCAPFile`_ keeps every octet after the header as the payload, so a - frame padded to the Ethernet minimum carries its padding there. The - default engine reads such octets as a trailer, outside the payload - (:issue:`1209`), and so does this (:issue:`1577`). - - .. _PyPCAPFile: https://github.com/kisom/pypcapfile - - """ - payload = _maybe_unhex(bytes(ipv4.payload)) - if not ipv4.len: - return payload - return payload[:max(ipv4.len - ipv4.hl * 4, 0)] - - -def _transport(ipv4: 'IP', depth: 'int' = 1) -> 'Optional[bytes]': - """Fetch the verbatim TCP segment carried by a PyPCAPFile IPv4 packet. - - Args: - ipv4: PyPCAPFile IPv4 packet. - depth: Its place in the frame (see :func:`_innermost`), by default the - first layer, as for a packet read on its own. - - Returns: - The raw TCP segment, as far as the Total Length (see - :func:`_ipv4_payload`), directly or behind AH or IPv6 extension headers - (see :func:`_past_extension_headers`, :issue:`1593`, :issue:`1597`), or - :data:`None` if the packet does - not carry a TCP header the default engine reads. A later fragment - carries none: from its offset on, its data is a slice of the datagram's - payload, with no upper-layer header in it (:rfc:`791#section-3.2`, - :issue:`1576`). Nor does a segment too short for a header, or whose Data - Offset is below 5 words (:issue:`1590`), or one past - :data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of the - frame, counting each header stepped over (see - :func:`_walk_ipv4_payload`, :issue:`1609`). - - Behind those, the default engine reads TCP only as its TCP parser does, so - the whole of the header its Data Offset gives must be there, and any - options must be ones that parser accepts (see - :func:`_default_accepts`). Directly over IPv4 it reads a header the - parser rejects out of the raw octets left behind instead - (:func:`pcapkit.toolkit.pcap.tcp_segment`, :issue:`1518`). - - """ - if (ipv4.p != Enum_TransType.TCP and ipv4.p not in IPV4_EXTENSION_HEADERS) or ipv4.off: - return None - if not _is_raw(ipv4.payload): - return None - - upper = _walk_ipv4_payload(ipv4.p, _ipv4_payload(ipv4), depth) - if upper is None or upper[0] != Enum_TransType.TCP: - return None - segment = upper[1] - if len(segment) < TCP_MIN_HEADER_LEN: - return None - header = (segment[12] >> 4) * 4 # the Data Offset, in octets - if header < TCP_MIN_HEADER_LEN: - return None - if ipv4.p != Enum_TransType.TCP and (header > len(segment) or ( - header > TCP_MIN_HEADER_LEN and not _default_accepts('TCP', segment[:header]))): - return None - return segment - - def _ethernet2dict(link: 'Any') -> 'dict[str, Any]': """Convert a PyPCAPFile Ethernet frame into :obj:`dict`. @@ -1382,13 +762,9 @@ def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Ad Data for IPv4 reassembly. * If the ``packet`` can be used for IPv4 reassembly. A packet can be reassembled - if it contains an IPv4 layer (:class:`pcapfile.protocols.network.ip.IP`), behind - 802.1Q or 802.1ad VLAN tags or not, or in a frame of one of the - :data:`READ_LINKTYPES`, and the **DF** flag is :data:`False`. The - layer is the first IPv4 packet of the frame, as for the default engine: one - carried in IPv6 counts (see :func:`_ipv4_in_ipv6`), and one whose header the - default engine rejects, or past its frame layer bound, does not (see - :func:`_ipv4_accepted` and :func:`_network_layer`). + if it contains an IPv4 layer (:class:`pcapfile.protocols.network.ip.IP`) whose + header was captured whole (see :func:`_network`), and the **DF** flag is + :data:`False`. * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for IPv4 reassembly (:term:`reasm.ipv4.packet`) will be returned; otherwise, returns :data:`None`. @@ -1400,9 +776,7 @@ def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Ad :class:`pcapkit.foundation.reassembly.ipv4.IPv4` """ - ipv4 = _network(packet, count) - if ipv4 is None: - ipv4 = _ipv4_in_ipv6(packet) + ipv4 = _network(packet) if ipv4 is None: return None if ipv4.flags & IPV4_FLAG_DF: # dismiss not fragmented packet @@ -1456,12 +830,8 @@ def tcp_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'TCP_Packet | None Data for TCP reassembly. * If the ``packet`` can be used for TCP reassembly. A packet can be reassembled - if it contains an IPv4 layer carrying a TCP segment, directly or behind AH or - IPv6 extension headers, and behind 802.1Q or 802.1ad VLAN tags or not, or in a - frame of one of the :data:`READ_LINKTYPES`, within the frame layer bound (see - :func:`_transport`). In an IP-in-IP tunnel the IPv4 layer is the innermost - IP layer, whose addresses key the segment (see :func:`_innermost`); TCP over - IPv6, tunnelled or not, cannot be, as `PyPCAPFile`_ has no IPv6 decoder. + if it contains an IPv4 layer carrying a TCP segment (see :func:`_transport`); + TCP over IPv6 cannot be, as `PyPCAPFile`_ has no IPv6 decoder. * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP reassembly (:term:`reasm.tcp.packet`) will be returned; otherwise, returns :data:`None`. @@ -1470,11 +840,11 @@ def tcp_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'TCP_Packet | None IPv4 address. Warns: - AttributeWarning: If ``packet`` carries TCP over IPv6, tunnelled in IP or - not, which is left out, and is the first such frame of its capture to - reach either :func:`tcp_reassembly` or :func:`tcp_traceflow`. The - rest of that capture's such frames are left out without another - warning. + AttributeWarning: If ``packet`` is an IPv6 packet straight over Ethernet + whose Next Header is TCP (see :func:`_ipv6_carries_tcp`), which is + left out, and is the first such frame of its capture to reach either + :func:`tcp_reassembly` or :func:`tcp_traceflow`. The rest of that + capture's such frames are left out without another warning. See Also: :class:`pcapkit.foundation.reassembly.tcp.TCP` @@ -1482,9 +852,13 @@ def tcp_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'TCP_Packet | None .. _PyPCAPFile: https://github.com/kisom/pypcapfile """ - ipv4, upper, depth = _innermost(packet, count) - if ipv4 is None or (segment := _transport(ipv4, depth)) is None: - _decline_tcp(packet, upper, count) + ipv4 = _network(packet) + if ipv4 is None: + _decline_ipv6_tcp(packet, count) + return None + + segment = _transport(ipv4) + if segment is None: return None # NOTE: imported only once there is something to decode, so that declining a @@ -1530,12 +904,8 @@ def tcp_traceflow(packet: 'Packet', *, data_link: 'Enum_LinkType', Data for TCP flow tracing. * If the ``packet`` can be used for TCP flow tracing. A packet can be traced - if it contains an IPv4 layer carrying a TCP segment, directly or behind AH or - IPv6 extension headers, and behind 802.1Q or 802.1ad VLAN tags or not, or in a - frame of one of the :data:`READ_LINKTYPES`, within the frame layer bound (see - :func:`_transport`). In an IP-in-IP tunnel the IPv4 layer is the innermost - IP layer, whose addresses key the segment (see :func:`_innermost`); TCP over - IPv6, tunnelled or not, cannot be, as `PyPCAPFile`_ has no IPv6 decoder. + if it contains an IPv4 layer carrying a TCP segment (see :func:`_transport`); + TCP over IPv6 cannot be, as `PyPCAPFile`_ has no IPv6 decoder. * If the ``packet`` can be traced, then the :obj:`dict` mapping of data for TCP flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`. @@ -1544,11 +914,11 @@ def tcp_traceflow(packet: 'Packet', *, data_link: 'Enum_LinkType', IPv4 address. Warns: - AttributeWarning: If ``packet`` carries TCP over IPv6, tunnelled in IP or - not, which is left out, and is the first such frame of its capture to - reach either :func:`tcp_reassembly` or :func:`tcp_traceflow`. The - rest of that capture's such frames are left out without another - warning. + AttributeWarning: If ``packet`` is an IPv6 packet straight over Ethernet + whose Next Header is TCP (see :func:`_ipv6_carries_tcp`), which is + left out, and is the first such frame of its capture to reach either + :func:`tcp_reassembly` or :func:`tcp_traceflow`. The rest of that + capture's such frames are left out without another warning. See Also: :class:`pcapkit.foundation.traceflow.tcp.TCP` @@ -1556,9 +926,13 @@ def tcp_traceflow(packet: 'Packet', *, data_link: 'Enum_LinkType', .. _PyPCAPFile: https://github.com/kisom/pypcapfile """ - ipv4, upper, depth = _innermost(packet, count) - if ipv4 is None or (segment := _transport(ipv4, depth)) is None: - _decline_tcp(packet, upper, count) + ipv4 = _network(packet) + if ipv4 is None: + _decline_ipv6_tcp(packet, count) + return None + + segment = _transport(ipv4) + if segment is None: return None # NOTE: imported only once there is something to decode, so that declining a diff --git a/pcapkit/toolkit/scapy.py b/pcapkit/toolkit/scapy.py index 4dc761fc64..293f3cc456 100644 --- a/pcapkit/toolkit/scapy.py +++ b/pcapkit/toolkit/scapy.py @@ -369,8 +369,8 @@ def _ipv4_accepted(ipv4: 'IP') -> 'bool': Note: `Scapy`_ dissects what an IPv4 EtherType or tunnel carries as IPv4 whatever its version or IHL, and never checks its options, so the - header is asked of the parser itself, as the pypcapfile engine asks it - (see :func:`pcapkit.toolkit.pypcapfile._default_accepts`). Its octets + header is asked of the parser itself (see + :func:`pcapkit.toolkit.pypcapfile._default_accepts`). Its octets are those `Scapy`_ dissected the layer from, which ``bytes()`` gives back as they were, as far as the IHL goes, and no fewer than the 20 of the fixed header, as the default engine reads at least those. A header @@ -544,8 +544,7 @@ def _ipv6_chain(ipv6: 'IPv6', depth: 'int') -> '_Chain': depth: Its place in the frame (see :func:`_network`). Returns: - The chain, walked over the octets by the default engine's rules, as - :func:`pcapkit.toolkit.pypcapfile._upper_layer` walks it: the + The chain, walked over the octets by the default engine's rules: the :data:`~pcapkit.toolkit.pypcapfile.IPV6_EXTENSION_HEADERS`, each sized by its own length field, up to :data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` of @@ -806,8 +805,7 @@ def _tcp_behind_extension_headers(segment: 'bytes') -> 'tuple[bytes, bytes] | No Directly over IP, the default engine reads a TCP header its parser rejects out of the raw octets left behind (:func:`pcapkit.toolkit.pcap.tcp_segment`), but behind another header - it does not, so the parser's answer stands here, as for the pypcapfile - engine. + it does not, so the parser's answer stands here. """ hdr_len = (segment[12] >> 4) * 4 if len(segment) >= TCP_MIN_HEADER_LEN else 0 diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index bbbd63dd01..152d623814 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -352,7 +352,7 @@ class Exclusion(NamedTuple): 'real coverage bought with exactly the false confidence #745 exists to ' "remove.\n\n" "#751's ruling was to take that trade: engine-tests installs PyPCAPFile on its " - 'own PyPCAPFile cell, covering the 9 HAS_PYPCAPFILE methods in ' + 'own PyPCAPFile cell, covering the 10 HAS_PYPCAPFILE methods in ' 'test_pypcapfile_unit.py (plus test_pypcapfile_engine.py) genuinely on every ' "one of the matrix's Python versions rather than only two of five -- #849's " "rebuild is exactly what closes the partial-coverage problem this paragraph " diff --git a/tests/foundation/engines/test_pypcapfile_engine.py b/tests/foundation/engines/test_pypcapfile_engine.py index 5126a22a0a..64a7da1dd5 100644 --- a/tests/foundation/engines/test_pypcapfile_engine.py +++ b/tests/foundation/engines/test_pypcapfile_engine.py @@ -5,9 +5,9 @@ gating, the IPv6 capability gap, per-frame decoding and its failure path, output, reassembly, tracing, storage -- are covered whether or not `pypcapfile`_ is installed. One test runs the real `pypcapfile`_ where it imports, over savefiles -built in memory, to pin the engine's record reader to the ``default`` engine's -reading in both byte orders (#1512, #1571). End-to-end agreement with the ``default`` engine lives in -:mod:`tests.foundation.engines.test_new_engine_parity`. +built in memory, to pin the engine's record reader to every record of the +savefile in both byte orders (#1512, #1571). End-to-end runs of the engine live in +:mod:`tests.foundation.engines.test_new_engine_parity_runtime`. .. _pypcapfile: https://github.com/kisom/pypcapfile @@ -123,10 +123,10 @@ def build_savefile(records, *, byteorder: str, nanosecond: bool = False, def reference_records(data: bytes) -> list: - """Every record of a savefile, read with :mod:`struct` alone, as the ``default`` - engine reads them: to a tail too short for a record header, keeping a record - whose capture length runs past the data as ``(seconds, fraction, captured - length, original length, the octets present)``.""" + """Every record of a savefile, read with :mod:`struct` alone: to a tail too + short for a record header, keeping a record whose capture length runs past the + data as ``(seconds, fraction, captured length, original length, the octets + present)``.""" byteorder, _ = next(key for key, magic in MAGIC.items() if magic == data[:4]) order = '<' if byteorder == 'little' else '>' records, offset = [], 24 @@ -301,24 +301,6 @@ def test_run_warns_and_gives_up_decoding_for_an_unknown_link_layer(self) -> None self.assertIn('unrecognised link layer protocol', warn.call_args.args[0]) self.assertIs(warn.call_args.args[1], AttributeWarning) - def test_run_warns_only_of_a_link_layer_left_unread(self) -> None: - # C.f. #1667: pcapfile decodes none of NULL, LOOP, RAW, IPV4 and IPV6, - # but the toolkit reads the IP packet in their frames, as the default - # engine does, so nothing is lost to warn of. Every other link type it - # has no decoder for, e.g. PPP (9), 12, 105, LINUX_SLL and 147, stays unread. - for linktype, unread in ((0, False), (108, False), (101, False), (228, False), (229, False), - (9, True), (12, True), (105, True), (113, True), (147, True), - (1, True)): - with self.subTest(linktype=linktype): - extractor, _ = self.make_extractor() - engine = self.engine(extractor, savefile=FakeSaveFile(ll_type=linktype), decoder=None) - with mock.patch('pcapkit.foundation.engines.pypcapfile.warn') as warn: - engine.run() - self.assertIsNone(engine._declf) - messages = [call.args[0] for call in warn.call_args_list] - self.assertEqual(any('unrecognised link layer protocol' in message for message in messages), - unread, messages) - def test_run_survives_the_malformed_upstream_link_layer_table(self) -> None: # ``pcapfile.linklayer.__LL_TYPES__`` carries a three-element entry for # LINKTYPE_IEEE802_11_RADIOTAP, so ``clookup`` raises IndexError for it. @@ -415,8 +397,8 @@ def test_a_nanosecond_savefile_keeps_its_resolution(self) -> None: 1500000000 + 123456789 / 1_000_000_000) def test_a_zero_length_record_is_read_and_left_undecoded(self) -> None: - """pcapfile ends the savefile at a zero-length record (#1571); the ``default`` - engine reads it, and every record after it, without a warning.""" + """pcapfile ends the savefile at a zero-length record (#1571); the engine + reads it, and every record after it, without a warning.""" cases = { 'middle': [(1, 0, b'first', 5), (2, 0, b'', 0), (3, 0, b'third', 5)], 'first': [(1, 0, b'', 0), (2, 0, b'second', 6), (3, 0, b'third', 5)], @@ -438,8 +420,8 @@ def test_a_zero_length_record_is_read_and_left_undecoded(self) -> None: def test_a_truncated_final_record_is_kept_as_captured(self) -> None: """pcapfile drops a final record whose capture length runs past the data - (#1571); the ``default`` engine keeps it, as declared, with the octets that - are there, and warns.""" + (#1571); the engine keeps it, as declared, with the octets that are there, + and warns.""" from pcapkit.utilities.warnings import ProtocolWarning for byteorder in ('little', 'big'): @@ -469,7 +451,7 @@ def test_a_tail_too_short_for_a_record_header_ends_the_savefile(self) -> None: self.assertEqual(warned, []) @unittest.skipUnless(HAS_PYPCAPFILE, 'pypcapfile not installed, or not importable here') - def test_records_agree_with_the_default_engine_against_the_real_pcapfile(self) -> None: + def test_every_record_is_read_against_the_real_pcapfile(self) -> None: """Against the real :mod:`pcapfile`, in both byte orders and resolutions: the engine reads every record :func:`reference_records` reads, field for field and octet for octet, into :mod:`pcapfile`'s own packet objects -- diff --git a/tests/toolkit/test_dpkt_pypcapfile_timestamp_rounding_1668_unit.py b/tests/toolkit/test_dpkt_pypcapfile_timestamp_rounding_1668_unit.py index e9305ae262..327365cd10 100644 --- a/tests/toolkit/test_dpkt_pypcapfile_timestamp_rounding_1668_unit.py +++ b/tests/toolkit/test_dpkt_pypcapfile_timestamp_rounding_1668_unit.py @@ -12,9 +12,8 @@ context of 12 digits gave ``1500000000.12``, and of 10 ``1500000000.0``. Every capture is built here, at ``ts_sec=1`` or ``1500000000``, so the module is -in the unit tier. The pypcapfile engine imports on Python 3.11 only and is -skipped elsewhere, but :func:`~pcapkit.toolkit.pypcapfile.packet2timestamp` is -checked on a stand-in packet everywhere. +in the unit tier. :func:`~pcapkit.toolkit.pypcapfile.packet2timestamp` is +checked on a stand-in packet, so needs no pypcapfile. """ from __future__ import annotations @@ -246,20 +245,6 @@ class DPKTTimestampAgreementTests(_Agreement): ENGINE = 'dpkt' -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -@unittest.skipUnless(pypcapfile_unit.HAS_PYPCAPFILE, 'pypcapfile not importable') -class PyPCAPFileTimestampAgreementTests(_Agreement): - """pypcapfile hands over the default engine's timestamp. - - It reads no PCAP-NG, and has no IPv6 reassembly. - - """ - - ENGINE = 'pypcapfile' - KINDS = ('ipv4', 'tcp', 'trace') - CONTAINERS = ('pcap',) - - @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') @unittest.skipUnless(HAS_DPKT, 'dpkt not installed') class DPKTAdapterTimestampTests(unittest.TestCase): diff --git a/tests/toolkit/test_pypcapfile_ext_chain_limit_1604_unit.py b/tests/toolkit/test_pypcapfile_ext_chain_limit_1604_unit.py deleted file mode 100644 index 2e3495936a..0000000000 --- a/tests/toolkit/test_pypcapfile_ext_chain_limit_1604_unit.py +++ /dev/null @@ -1,149 +0,0 @@ -""":mod:`pcapkit.toolkit.pypcapfile` stops a chain of IPv6 extension headers where the default engine does. - -GitHub issue #1604: the default engine dissects at most -:data:`~pcapkit.protocols.internet.internet.EXTENSION_HEADER_LIMIT` extension -headers in a row and leaves the rest raw, so the TCP behind a longer chain is -not found. :func:`~pcapkit.toolkit.pypcapfile._past_extension_headers` (inside -IPv4) and :func:`~pcapkit.toolkit.pypcapfile._upper_layer` (over IPv6) walk such -a chain in a loop, and stop at the same depth, so the TCP the toolkit reads and -the TCP it warns of leaving out are the TCP the default engine finds. - -These tests use stand-ins for `pypcapfile`_'s decoders, so run without it; the -comparison of the two engines over a capture is in -:mod:`tests.foundation.engines.test_ext_chain_limit_agreement_1604_unit`. - -.. _pypcapfile: https://github.com/kisom/pypcapfile - -""" -from __future__ import annotations - -import unittest -from unittest import mock - -from tests._support import reimport_once_per_class -from tests.foundation import _roundtrip as wire -from tests.toolkit import test_pypcapfile_unit as base - -#: Chain lengths either side of the limit, and one far past it. -AT_LIMIT, PAST_LIMIT = (32,), (33, 1000) - - -def chain(count: int, last: int, *, mixed: bool = False) -> bytes: - """``count`` extension headers, the last naming ``last``. - - Destination Options headers only, or alternately Destination Options and - AH when ``mixed``; the first is always Destination Options (protocol 60). - - """ - kinds = ['ah' if mixed and index % 2 else 'opts' for index in range(count)] - codes = [51 if kind == 'ah' else 60 for kind in kinds] + [last] - return b''.join(base.ah(nxt, 1) if kind == 'ah' else base.options_header(nxt) - for kind, nxt in zip(kinds, codes[1:])) - - -@unittest.skipUnless(base.HAS_RUNTIME, 'runtime dependencies not installed') -class PyPCAPFileExtensionHeaderChainLimitTests(unittest.TestCase): - """C.f. #1604.""" - - decline = base.PyPCAPFileTCPOverIPv6Tests.decline - - def setUp(self) -> None: - reimport_once_per_class(self) - - def test_transport_reads_tcp_behind_a_chain_of_32_inside_ipv4(self) -> None: - from pcapkit.toolkit.pypcapfile import _transport - - segment = base.make_tcp(b'data') - for count in AT_LIMIT: - for mixed in (False, True): - with self.subTest(count=count, mixed=mixed): - layer = base.FakeIP(chain(count, 6, mixed=mixed) + segment, p=60) - self.assertEqual(_transport(layer), segment) - - def test_no_tcp_behind_a_longer_chain_inside_ipv4(self) -> None: - from pcapkit.toolkit import pypcapfile - - segment = base.make_tcp(b'data') - for count in PAST_LIMIT: - for mixed in (False, True): - with self.subTest(count=count, mixed=mixed): - layer = base.FakeIP(chain(count, 6, mixed=mixed) + segment, p=60) - with mock.patch.object(pypcapfile, '_default_extension_header', - wraps=pypcapfile._default_extension_header) as parse: - self.assertIsNone(pypcapfile._transport(layer)) - # the walk stops at the limit, rather than reading the rest - self.assertLessEqual(parse.call_count, 32) - self.assertEqual(self.decline(base.make_packet(base.FakeEthernet(layer))), []) - - def test_tcp_over_ipv6_behind_a_chain_of_32_warns(self) -> None: - for count in AT_LIMIT: - for mixed in (False, True): - with self.subTest(count=count, mixed=mixed): - packet = wire.ipv6(chain(count, 6, mixed=mixed) + base.TCP_SEGMENT, nxt=60) - messages = self.decline(base.make_packet(base.ipv6_frame(packet))) - self.assertEqual(len(messages), 1, messages) - self.assertIn('TCP over IPv6', messages[0]) - - def test_tcp_over_ipv6_behind_a_longer_chain_does_not_warn(self) -> None: - # The default engine finds no TCP there, so none is left out. - for count in PAST_LIMIT: - for mixed in (False, True): - with self.subTest(count=count, mixed=mixed): - packet = wire.ipv6(chain(count, 6, mixed=mixed) + base.TCP_SEGMENT, nxt=60) - self.assertEqual(self.decline(base.make_packet(base.ipv6_frame(packet))), []) - - def test_a_tunnel_behind_the_chain_is_entered_only_within_the_limit(self) -> None: - # C.f. #1581: the TCP in a 4in4 tunnel is read from the inner IPv4, but - # only if the chain in front of the tunnel is within the limit. - from pcapkit.toolkit.pypcapfile import _innermost, _transport - - tcp4 = wire.ipv4(base.TCP_SEGMENT, proto=6) - for count, entered in ((32, True), (33, False), (1000, False)): - with self.subTest(count=count), \ - mock.patch.dict('sys.modules', base.fake_ip_decoder()) as modules: - IP = modules['pcapfile.protocols.network.ip'].IP - packet = base.make_packet(base.tunnel(chain(count, 4) + tcp4, 60)) - ipv4, upper, depth = _innermost(packet) - self.assertIsNone(upper) - self.assertEqual(IP.calls, [(tcp4, 0)] if entered else []) - self.assertEqual(_transport(ipv4, depth), base.TCP_SEGMENT if entered else None) - if not entered: # no TCP, and none left out either - self.assertEqual(self.decline(packet), []) - - def test_a_new_chain_inside_a_tunnel_is_counted_afresh(self) -> None: - # 20 headers, a tunnel, then 20 more: 40 in the packet, but never more - # than 20 in a row, so the TCP is found. 33 behind the tunnel are not - # read past, wherever the count starts. - tcp4 = base.TCP_SEGMENT - for count, found in ((20, True), (33, False)): - with self.subTest(tunnel='6in6', count=count), \ - mock.patch.dict('sys.modules', base.fake_ip_decoder()): - inner = wire.ipv6(chain(count, 6) + base.TCP_SEGMENT, nxt=60) - packet = wire.ipv6(chain(20, 41) + inner, nxt=60) - messages = self.decline(base.make_packet(base.ipv6_frame(packet))) - self.assertEqual(len(messages), int(found), messages) - if found: - self.assertTrue(messages[0].startswith('Frame 1: TCP over IPv6 is left out'), - messages) - with self.subTest(tunnel='4in4', count=count), \ - mock.patch.dict('sys.modules', base.fake_ip_decoder()): - from pcapkit.toolkit.pypcapfile import _innermost, _transport - - inner = wire.ipv4(chain(count, 6) + tcp4, proto=60) - ipv4, _, depth = _innermost(base.make_packet(base.tunnel(chain(20, 4) + inner, 60))) - self.assertEqual(_transport(ipv4, depth), tcp4 if found else None) - - def test_ipv4_in_ipv6_is_found_only_within_the_limit(self) -> None: - from pcapkit.toolkit.pypcapfile import _ipv4_in_ipv6 - - inner = base.udp_fragment() - for count, found in ((32, True), (33, False), (1000, False)): - with self.subTest(count=count), \ - mock.patch.dict('sys.modules', base.fake_ip_decoder()) as modules: - IP = modules['pcapfile.protocols.network.ip'].IP - frame = base.ipv6_frame(wire.ipv6(chain(count, 4) + inner, nxt=60)) - self.assertEqual(isinstance(_ipv4_in_ipv6(base.make_packet(frame)), IP), found) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/toolkit/test_pypcapfile_header_checks_unit.py b/tests/toolkit/test_pypcapfile_header_checks_unit.py deleted file mode 100644 index 71345829ff..0000000000 --- a/tests/toolkit/test_pypcapfile_header_checks_unit.py +++ /dev/null @@ -1,264 +0,0 @@ -"""The header checks :mod:`pcapkit.toolkit.pypcapfile` makes as the default engine does. - -`pypcapfile`_ slices an IPv4 header at its IHL however few octets were captured, -never parses its options, has no IPv6 decoder and no AH decoder, and reads a TCP -Data Offset below 5 words as if it were 5. The toolkit reads such frames as the -default engine does instead: GH#1590 (Data Offset below 5), GH#1591 (IHL past -the packet), GH#1592 (IPv4 in IPv6), GH#1593 (TCP behind an IPv4 AH header) and -GH#1596 (IPv4 options the default engine rejects). - -These tests use stand-ins for `pypcapfile`_'s decoders, so run without it. The -ones that read real captures with both engines need it, so live in -:mod:`tests.toolkit.test_pypcapfile_unit`, which the PyPCAPFile CI cell runs. - -.. _pypcapfile: https://github.com/kisom/pypcapfile - -""" -from __future__ import annotations - -import unittest -import warnings -from unittest import mock - -from tests._support import reimport_once_per_class -from tests.foundation import _roundtrip as wire -from tests.toolkit import test_pypcapfile_unit as base - - -@unittest.skipUnless(base.HAS_RUNTIME, 'runtime dependencies not installed') -class PyPCAPFileHeaderCheckTests(unittest.TestCase): - """Stand-ins for the decoded layers, so `pypcapfile`_ is not needed.""" - - decline = base.PyPCAPFileTCPOverIPv6Tests.decline - - def setUp(self) -> None: - reimport_once_per_class(self) - - def test_data_offset_below_5_is_no_tcp(self) -> None: - # C.f. #1590: no TCP input, and no warning either. - for words in range(5): - with self.subTest(words=words): - segment = base.data_offset(base.make_tcp(b'data'), words) - frame = base.FakeEthernet(base.FakeIP(segment)) - self.assertEqual(self.decline(base.make_packet(frame)), []) - - def test_ipv4_header_past_the_packet_is_no_ipv4(self) -> None: - # C.f. #1591: pypcapfile slices the 60-octet header of a 40-octet packet - # as 20 octets of options and no payload. - from pcapkit.toolkit.pypcapfile import ipv4_reassembly - - for label, layer in ( - ('udp', base.FakeIP(b'', p=17, hl=15, opt=b'\x01' * 20, len=40)), - ('tcp', base.FakeIP(b'', hl=15, opt=base.make_tcp(b''), len=40)), - ('one option word short', base.FakeIP(b'', p=17, hl=7, opt=b'\x01' * 4, len=24)), - ): - with self.subTest(case=label): - packet = base.make_packet(base.FakeEthernet(layer)) - self.assertIsNone(ipv4_reassembly(packet, count=1)) - self.assertEqual(self.decline(packet), []) - - def test_ipv4_options_the_default_engine_rejects_are_no_ipv4(self) -> None: - # C.f. #1596: a fragment with such options is no IPv4 input, a segment - # behind them no TCP input; one with options it accepts is both. - from pcapkit.toolkit.pypcapfile import ipv4_reassembly - - for options in base.BAD_IPV4_OPTIONS: - with self.subTest(options=options.hex()): - packet = base.make_packet(base.FakeEthernet(base.FakeIP(b'x' * 16, p=17, hl=6, - opt=options))) - self.assertIsNone(ipv4_reassembly(packet, count=1)) - tcp = base.make_packet(base.FakeEthernet(base.FakeIP(base.make_tcp(b'data'), hl=6, - opt=options))) - self.assertEqual(self.decline(tcp), []) - for options in base.GOOD_IPV4_OPTIONS: - with self.subTest(options=options.hex()): - packet = base.make_packet(base.FakeEthernet(base.FakeIP(b'x' * 16, p=17, hl=6, - opt=options))) - data = ipv4_reassembly(packet, count=1) - self.assertIsNotNone(data) - self.assertEqual(data.header[20:], options) - - def test_options_are_parsed_once_per_frame(self) -> None: - # The adapters each fetch a frame's network layer; the default engine's - # parser is asked once between them, and not at all without options. - from pcapkit.const.reg.linktype import LinkType - from pcapkit.toolkit import pypcapfile - - pypcapfile._default_accepts.cache_clear() - real = pypcapfile.Protocol_IPv4 - frames = [base.FakeEthernet(base.FakeIP(b'x' * 16, p=17, hl=6, opt=options)) - for options in (*base.GOOD_IPV4_OPTIONS, base.BAD_IPV4_OPTIONS[0])] - frames.append(base.FakeEthernet(base.FakeIP(b'x' * 16, p=17))) - with mock.patch.object(pypcapfile, 'Protocol_IPv4', side_effect=real) as parser: - for count, frame in enumerate(frames, start=1): - packet = base.make_packet(frame) - pypcapfile.ipv4_reassembly(packet, count=count) - pypcapfile.tcp_reassembly(packet, count=count) - pypcapfile.tcp_traceflow(packet, data_link=LinkType.ETHERNET, count=count) - self.assertEqual([call.args[0][20:] for call in parser.call_args_list], - [*base.GOOD_IPV4_OPTIONS, base.BAD_IPV4_OPTIONS[0]]) - - def test_parsing_options_leaves_the_warnings_filters_and_the_logger_level_alone(self) -> None: - import logging - - from pcapkit.toolkit.pypcapfile import ipv4_reassembly - - logger = logging.getLogger('pcapkit') - level, filters = logger.level, list(warnings.filters) - for count, options in enumerate((*base.BAD_IPV4_OPTIONS, *base.GOOD_IPV4_OPTIONS), start=1): - frame = base.FakeEthernet(base.FakeIP(b'x' * 16, p=17, hl=6, opt=options)) - ipv4_reassembly(base.make_packet(frame), count=count) - self.assertEqual(warnings.filters, filters) - self.assertEqual(logger.level, level) - - def test_ipv4_in_ipv6_is_found(self) -> None: - # C.f. #1592: the first IPv4 packet inside IPv6, however it is wrapped, - # goes to pypcapfile's own IPv4 decoder. - from pcapkit.toolkit.pypcapfile import _ipv4_in_ipv6 - - inner = base.udp_fragment() - for label, frame in ( - ('4in6', base.ipv6_frame(wire.ipv6(inner, nxt=4))), - ('802.1Q, 4in6', base.tagged_frame(wire.ipv6(inner, nxt=4), 0x86DD, base.C_TAG)), - ('hop-by-hop', base.ipv6_frame(wire.ipv6(bytes([4, 0, 1, 4, 0, 0, 0, 0]) + inner, nxt=0))), - ('first fragment', base.ipv6_frame(wire.ipv6_fragment(inner, mf=True, nxt=4))), - ('ah', base.ipv6_frame(wire.ipv6(base.ah(4) + inner, nxt=51))), - ('4in6in6', base.ipv6_frame(wire.ipv6(wire.ipv6(inner, nxt=4), nxt=41))), - ('ethernet padding', base.ipv6_frame(wire.ipv6(inner, nxt=4) + bytes(6))), - ): - with self.subTest(case=label), mock.patch.dict('sys.modules', base.fake_ip_decoder()) as modules: - IP = modules['pcapfile.protocols.network.ip'].IP - self.assertIsInstance(_ipv4_in_ipv6(base.make_packet(frame)), IP) - self.assertEqual(IP.calls, [(inner, 0)]) - - def test_no_ipv4_in_ipv6_where_the_default_engine_finds_none(self) -> None: - from pcapkit.toolkit.pypcapfile import _ipv4_in_ipv6 - - inner = base.udp_fragment() - for label, frame, decoded in ( - ('ipv4 over ethernet', base.FakeEthernet(base.FakeIP(inner, p=4)), False), - ('ipv6, udp', base.ipv6_frame(wire.ipv6(wire.udp(b'x'), nxt=17)), False), - ('ipv6 later fragment', base.ipv6_frame(wire.ipv6_fragment(inner, offset=8, nxt=4)), False), - ('ipv6 truncated', base.ipv6_frame(wire.ipv6(inner, nxt=4)[:39]), False), - ('esp', base.ipv6_frame(wire.ipv6(bytes(8) + inner, nxt=50)), False), - ('802.1Q, no ipv6 header', base.tagged_frame(b'', 0x86DD, base.C_TAG), False), - ('undecoded frame', b'\x00' * 20, False), - # pypcapfile never decodes IPv6, so has nothing to warn of - ('inner ihl 4', base.ipv6_frame(wire.ipv6(base.ihl(inner, 4), nxt=4)), True), - ('inner ihl past the packet', base.ipv6_frame(wire.ipv6(base.ihl(inner, 15), nxt=4)), True), - ): - with self.subTest(case=label), warnings.catch_warnings(record=True) as caught, \ - mock.patch.dict('sys.modules', base.fake_ip_decoder()) as modules: - warnings.simplefilter('always') - IP = modules['pcapfile.protocols.network.ip'].IP - self.assertIsNone(_ipv4_in_ipv6(base.make_packet(frame))) - self.assertEqual(bool(IP.calls), decoded) - self.assertEqual(caught, []) - - def test_transport_reads_tcp_behind_ah(self) -> None: - # C.f. #1593: the default engine dissects AH over IPv4, and the TCP behind it. - from pcapkit.toolkit.pypcapfile import _transport - - segment = base.make_tcp(b'data') - options = base.make_tcp(b'data', options=b'\x02\x04\x05\xb4') # MSS - for label, payload, want in ( - ('ah', base.ah(6) + segment, segment), - ('ah of the fixed part only', base.ah(6, 1) + segment, segment), - ('two ah', base.ah(51) + base.ah(6, 1) + segment, segment), - ('ah, tcp options', base.ah(6) + options, options), - ): - with self.subTest(case=label): - self.assertEqual(_transport(base.FakeIP(payload, p=51)), want) - - def test_no_tcp_behind_ah_where_the_default_engine_finds_none(self) -> None: - # Behind AH there is no fall-back to the raw octets (#1518), so a TCP - # header its parser rejects is no TCP -- and no TCP over IPv6 either. - from pcapkit.toolkit.pypcapfile import _transport - - segment = base.make_tcp(b'data') - for label, payload, fields in ( - ('payload length 0', base.ah(6, 0) + segment, {}), - ('ah cut short', base.ah(6)[:15], {}), - ('ah under the fixed part', base.ah(6)[:11], {}), - ('ah, udp', base.ah(17) + wire.udp(b'x'), {}), - ('ah, tcp truncated', base.ah(6) + segment[:19], {}), - ('ah, data offset 4', base.ah(6) + base.data_offset(segment, 4), {}), - ('ah, data offset past the capture', base.ah(6) + base.data_offset(segment, 9), {}), - ('ah, mss option of length 3', base.ah(6) + base.make_tcp(b'', options=b'\x02\x03\x00\x00'), {}), - ('later fragment', base.ah(6) + segment, {'off': 3}), - ): - with self.subTest(case=label): - layer = base.FakeIP(payload, p=51, **fields) - self.assertIsNone(_transport(layer)) - self.assertEqual(self.decline(base.make_packet(base.FakeEthernet(layer))), []) - - def test_transport_reads_tcp_behind_ipv6_extension_headers(self) -> None: - # C.f. #1597: inside IPv4 too, the default engine dissects these headers - # and what each one's Next Header names. - from pcapkit.toolkit.pypcapfile import _transport - - segment = base.make_tcp(b'data') - cases = [(label, protocol, header) for label, protocol, header in base.IPV4_EXTENSION_HEADERS] - cases += [('hop-by-hop, destination options', 0, base.options_header(60) + base.options_header(6)), - ('destination options, ah', 60, base.options_header(51) + base.ah(6)), - ('ah, hop-by-hop', 51, base.ah(0) + base.options_header(6))] - for label, protocol, header in cases: - with self.subTest(case=label): - self.assertEqual(_transport(base.FakeIP(header + segment, p=protocol)), segment) - - def test_no_tcp_behind_ipv6_extension_headers_where_the_default_engine_finds_none(self) -> None: - from pcapkit.toolkit.pypcapfile import _transport - - segment = base.make_tcp(b'data') - for label, protocol, payload in ( - ('padn of length 9', 60, base.options_header(6, b'\x01\x09' + bytes(4)) + segment), - ('header length past the packet', 60, bytes([6, 9]) + bytes(6) + segment), - ('cut short', 0, base.options_header(6)[:4]), - # read as raw octets, as fewer were captured than the header needs - ('routing header cut short', 43, bytes([6, 0, 0, 0])), - ('routing type 3, under its addresses', 43, bytes([6, 0, 3, 1]) + bytes(4) + segment), - ('hip header length 3', 139, bytes([6, 3, 0x01, 0x21]) + bytes(28) + segment), - ('shim6, whose parser refuses ipv4', 140, bytes([6, 0, 0x80, 0]) + bytes(4) + segment), - ('no next header', 59, segment), - ('tcp option of length 3', 60, - base.options_header(6) + base.make_tcp(b'', options=b'\x02\x03\x00\x00')), - ): - with self.subTest(case=label): - layer = base.FakeIP(payload, p=protocol) - self.assertIsNone(_transport(layer)) - self.assertEqual(self.decline(base.make_packet(base.FakeEthernet(layer))), []) - - def test_extension_headers_are_parsed_once_per_frame(self) -> None: - from pcapkit.const.reg.linktype import LinkType - from pcapkit.toolkit import pypcapfile - - pypcapfile._default_extension_header.cache_clear() - layer = base.FakeIP(base.options_header(60) + base.options_header(17) + wire.udp(b'x'), p=0) - packet = base.make_packet(base.FakeEthernet(layer)) - pypcapfile.ipv4_reassembly(packet, count=1) - pypcapfile.tcp_reassembly(packet, count=1) - pypcapfile.tcp_traceflow(packet, data_link=LinkType.ETHERNET, count=1) - self.assertEqual(pypcapfile._default_extension_header.cache_info().misses, 2) - - def test_tunnelled_tcp_behind_ah_is_found(self) -> None: - # C.f. #1581: a tunnel behind AH or an IPv6 extension header in IPv4 is - # walked into, and the TCP read from the innermost IPv4 layer. - from pcapkit.toolkit.pypcapfile import _innermost, _transport - - tcp4 = wire.ipv4(base.TCP_SEGMENT, proto=6) - inner_ah = wire.ipv4(base.ah(6) + base.TCP_SEGMENT, proto=51) - for label, frame, inner in ( - ('4in4 behind ah', base.tunnel(base.ah(4) + tcp4, 51), tcp4), - ('4in4, inner ah', base.tunnel(inner_ah, 4), inner_ah), - ('4in4 behind destination options', base.tunnel(base.options_header(4) + tcp4, 60), tcp4), - ): - with self.subTest(case=label), mock.patch.dict('sys.modules', base.fake_ip_decoder()) as modules: - IP = modules['pcapfile.protocols.network.ip'].IP - ipv4, _, depth = _innermost(base.make_packet(frame)) - self.assertIsInstance(ipv4, IP) - self.assertEqual(IP.calls, [(inner, 0)]) - self.assertEqual(_transport(ipv4, depth), base.TCP_SEGMENT) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/toolkit/test_pypcapfile_linktypes_and_layer_bound_1609_1667_unit.py b/tests/toolkit/test_pypcapfile_linktypes_and_layer_bound_1609_1667_unit.py deleted file mode 100644 index f7ebaac68d..0000000000 --- a/tests/toolkit/test_pypcapfile_linktypes_and_layer_bound_1609_1667_unit.py +++ /dev/null @@ -1,345 +0,0 @@ -""":mod:`pcapkit.toolkit.pypcapfile` reads the link types, and stops at the frame depth, the default engine does. - -GitHub issue #1667: `pypcapfile`_ 0.12.0 decodes Ethernet alone, so the engine -left every frame of the ``NULL``, ``LOOP``, ``RAW``, ``IPV4`` and ``IPV6`` link -types undecoded, and the adapters handed nothing to TCP reassembly, flow tracing -or IPv4 reassembly, where the default engine reads the IP packet in them. They -now read it themselves, and :func:`~pcapkit.toolkit.pypcapfile.packet2dict` -reports an unread frame's octets rather than their hex text. - -GitHub issue #1609: the default engine dissects at most -:data:`~pcapkit.protocols.protocol.FRAME_LAYER_LIMIT` layers of a frame, each -VLAN tag, IP header, AH and IPv6 extension header one of them. The adapters' -walks ran past it, and now stop there too. - -These use stand-ins for `pypcapfile`_'s IP and TCP decoders, so run without it; -the two engines are compared over real captures in -:mod:`tests.foundation.engines.test_pypcapfile_linktype_and_layer_bound_agreement_1609_1667_unit`. - -.. _pypcapfile: https://github.com/kisom/pypcapfile - -""" -from __future__ import annotations - -import binascii -import ipaddress -import unittest -from unittest import mock - -from tests._support import reimport_once_per_class -from tests.foundation import _roundtrip as wire -from tests.toolkit import test_pypcapfile_tcp_last_unit as tcp_last -from tests.toolkit import test_pypcapfile_unit as base - -#: ``LINKTYPE_*`` numbers. -NULL, ETHERNET, RAW, LOOP, IPV4, IPV6, LINUX_SLL = 0, 1, 101, 108, 228, 229, 113 - -#: An IPv4 packet carrying :data:`~tests.toolkit.test_pypcapfile_unit.TCP_SEGMENT`, -#: 47 octets long, and an IPv6 one. -TCP4 = wire.ipv4(base.TCP_SEGMENT, proto=6) -TCP6 = wire.ipv6(base.TCP_SEGMENT, nxt=6) -#: A first IPv4 fragment of a UDP datagram. -FRAGMENT = base.udp_fragment() -#: The addresses :mod:`tests.foundation._roundtrip` gives a packet. -SRC, DST = ipaddress.IPv4Address(wire.IPV4_A), ipaddress.IPv4Address(wire.IPV4_B) - - -def stand_ins() -> dict: - """Stand-in :mod:`pcapfile` modules, for :data:`sys.modules`: its IP and its TCP decoder.""" - modules = base.fake_ip_decoder() - for name, module in tcp_last.fake_pcapfile().items(): - modules.setdefault(name, module) - return modules - - -def unread(frame: bytes, linktype: int): - """A frame of a capture of ``linktype``, as the engine hands one over that `pypcapfile`_ cannot decode.""" - return base.make_packet(binascii.hexlify(frame), linktype=linktype) - - -def loopback(packet: bytes, family: int, byteorder: str) -> bytes: - """``packet`` behind a BSD loopback header of address family ``family``.""" - return family.to_bytes(4, byteorder) + packet - - -def nest4(inner: bytes, count: int, proto: int) -> bytes: - """``count`` IPv4 headers, the innermost carrying ``inner`` as protocol ``proto``, each outer one IPv4.""" - packet = wire.ipv4(inner, proto=proto) - for _ in range(count - 1): - packet = wire.ipv4(packet, proto=4) - return packet - - -def nest6(inner: bytes, count: int, nxt: int) -> bytes: - """``count`` IPv6 headers, the innermost carrying ``inner`` as ``nxt``, each outer one IPv6.""" - packet = wire.ipv6(inner, nxt=nxt) - for _ in range(count - 1): - packet = wire.ipv6(packet, nxt=41) - return packet - - -def ethernet(packet: bytes): - """``packet``, an IPv4 packet with no options, behind Ethernet, as `pypcapfile`_ decodes it.""" - return base.make_packet(base.tunnel(packet[20:], packet[9])) - - -def ethernet6(packet: bytes): - """``packet``, an IPv6 packet, behind Ethernet, as `pypcapfile`_ leaves it.""" - return base.make_packet(base.ipv6_frame(packet)) - - -def ah_pairs(count: int) -> bytes: - """``count`` pairs of an IPv4 header and an AH, then the TCP segment.""" - packet = base.TCP_SEGMENT - for index in range(count): - packet = wire.ipv4(base.ah(6 if index == 0 else 4) + packet, proto=51) - return packet - - -def hbh_pairs(count: int) -> bytes: - """``count`` pairs of an IPv6 header and a Hop-by-Hop Options header, then the TCP segment.""" - packet = base.TCP_SEGMENT - for index in range(count): - packet = wire.ipv6(base.options_header(6 if index == 0 else 41) + packet, nxt=0) - return packet - - -def tags(packet: bytes, ethertype: int, count: int, tpid: int): - """``packet`` behind ``count`` VLAN tags of ``tpid``, as `pypcapfile`_ leaves it.""" - return base.make_packet(base.tagged_frame(packet, ethertype, *(tpid,) * count)) - - -def chain(count: int, last: int) -> bytes: - """``count`` Destination Options headers, the last naming ``last``.""" - return b''.join(base.options_header(60 if index < count - 1 else last) for index in range(count)) - - -@unittest.skipUnless(base.HAS_RUNTIME, 'runtime dependencies not installed') -class PyPCAPFileLinkTypeTests(unittest.TestCase): - """C.f. #1667.""" - - decline = base.PyPCAPFileTCPOverIPv6Tests.decline - - def setUp(self) -> None: - reimport_once_per_class(self) - - def test_tcp_is_read_over_each_link_type_the_default_engine_dissects(self) -> None: - from pcapkit.const.reg.linktype import LinkType - from pcapkit.toolkit.pypcapfile import tcp_reassembly, tcp_traceflow - - tcp4in6 = wire.ipv6(TCP4, nxt=4) - for label, linktype, frame, read in ( - ('NULL, little-endian', NULL, loopback(TCP4, 2, 'little'), True), - ('NULL, big-endian', NULL, loopback(TCP4, 2, 'big'), True), - ('NULL, 4in6 over AF_INET6 28', NULL, loopback(tcp4in6, 28, 'little'), True), - ('NULL, 4in6 over AF_INET6 30', NULL, loopback(tcp4in6, 30, 'little'), True), - ('LOOP', LOOP, loopback(TCP4, 2, 'big'), True), - ('LOOP, 4in6', LOOP, loopback(tcp4in6, 24, 'big'), True), - ('RAW', RAW, TCP4, True), - ('RAW, 4in4', RAW, wire.ipv4(TCP4, proto=4), True), - ('RAW, 4in6', RAW, tcp4in6, True), - ('RAW, behind AH', RAW, wire.ipv4(base.ah(6) + base.TCP_SEGMENT, proto=51), True), - ('IPV4', IPV4, TCP4, True), - ('IPV6, 4in6', IPV6, tcp4in6, True), - # where the default engine dissects no IP either - ('LOOP, family written little-endian', LOOP, loopback(TCP4, 2, 'little'), False), - ('NULL, family 7', NULL, loopback(TCP4, 7, 'little'), False), - ('NULL, cut short of its header', NULL, b'\x02\x00\x00', False), - ('RAW, version 5', RAW, b'\x55' + TCP4[1:], False), - ('RAW, empty', RAW, b'', False), - ('IPV4, an IPv6 packet', IPV4, TCP6, False), - ('12', 12, TCP4, False), - ('LINUX_SLL', LINUX_SLL, TCP4, False), - ): - with self.subTest(case=label), mock.patch.dict('sys.modules', stand_ins()): - packet = unread(frame, linktype) - data = tcp_reassembly(packet, count=1) - trace = tcp_traceflow(packet, data_link=LinkType.get(linktype), count=1) - if not read: - self.assertIsNone(data) - self.assertIsNone(trace) - self.assertEqual(self.decline(packet), []) - continue - self.assertIsNotNone(data) - self.assertEqual(data.bufid, (SRC, 40000, DST, 9)) - self.assertEqual((data.header, data.payload), (base.TCP_SEGMENT[:20], bytearray(b'payload'))) - self.assertEqual((trace.src, trace.dst, trace.srcport, trace.dstport), (SRC, DST, 40000, 9)) - self.assertEqual(trace.header, base.TCP_SEGMENT[:20]) - - def test_ipv4_fragments_are_read_over_each_link_type_the_default_engine_dissects(self) -> None: - from pcapkit.const.reg.transtype import TransType - from pcapkit.toolkit.pypcapfile import ipv4_reassembly - - in6 = wire.ipv6(FRAGMENT, nxt=4) - for label, linktype, frame, read in ( - ('NULL', NULL, loopback(FRAGMENT, 2, 'little'), True), - ('NULL, 4in6', NULL, loopback(in6, 24, 'little'), True), - ('LOOP', LOOP, loopback(FRAGMENT, 2, 'big'), True), - ('LOOP, 4in6', LOOP, loopback(in6, 30, 'big'), True), - ('RAW', RAW, FRAGMENT, True), - ('RAW, 4in6', RAW, in6, True), - ('IPV4', IPV4, FRAGMENT, True), - ('IPV6, 4in6', IPV6, in6, True), - ('IPV6, an IPv4 packet', IPV6, FRAGMENT, False), - ('RAW, cut short of the header', RAW, FRAGMENT[:15], False), - ('12', 12, FRAGMENT, False), - ('LINUX_SLL', LINUX_SLL, FRAGMENT, False), - ): - with self.subTest(case=label), mock.patch.dict('sys.modules', stand_ins()): - data = ipv4_reassembly(unread(frame, linktype), count=3) - if not read: - self.assertIsNone(data) - continue - self.assertIsNotNone(data) - self.assertEqual(data.bufid, (SRC, DST, 0x1234, TransType.UDP)) - self.assertEqual((data.num, data.fo, data.mf, data.tl), (3, 0, True, len(FRAGMENT))) - self.assertEqual((data.header, data.payload), (FRAGMENT[:20], bytearray(FRAGMENT[20:]))) - - def test_tcp_over_ipv6_is_left_out_with_a_warning_over_each_link_type(self) -> None: - for label, linktype, frame in ( - ('NULL, AF_INET6 24', NULL, loopback(TCP6, 24, 'little')), - ('NULL, AF_INET6 28', NULL, loopback(TCP6, 28, 'little')), - ('NULL, AF_INET6 30, big-endian', NULL, loopback(TCP6, 30, 'big')), - ('LOOP', LOOP, loopback(TCP6, 24, 'big')), - ('RAW', RAW, TCP6), - ('RAW, 6in4', RAW, wire.ipv4(TCP6, proto=41)), - ('IPV6', IPV6, TCP6), - ): - with self.subTest(case=label), mock.patch.dict('sys.modules', stand_ins()): - messages = self.decline(unread(frame, linktype)) - self.assertEqual(len(messages), 1, messages) - self.assertTrue(messages[0].startswith('Frame 1: TCP over IPv6 is left out'), messages) - - def test_packet2dict_reports_an_unread_frames_octets(self) -> None: - from pcapkit.const.reg.linktype import LinkType - from pcapkit.toolkit.pypcapfile import packet2dict - - self.assertEqual(len(TCP4), 47) - for linktype in (RAW, LINUX_SLL, ETHERNET): - with self.subTest(linktype=linktype): - link = LinkType.get(linktype) - info = packet2dict(unread(TCP4, linktype), data_link=link) - self.assertEqual(info[link.name], {'raw_len': 47, 'raw': TCP4}) - - -@unittest.skipUnless(base.HAS_RUNTIME, 'runtime dependencies not installed') -class PyPCAPFileFrameLayerBoundTests(unittest.TestCase): - """C.f. #1609.""" - - decline = base.PyPCAPFileTCPOverIPv6Tests.decline - - def setUp(self) -> None: - reimport_once_per_class(self) - - def test_tcp_is_read_only_within_the_frame_layer_bound(self) -> None: - from pcapkit.const.reg.linktype import LinkType - from pcapkit.toolkit.pypcapfile import tcp_reassembly, tcp_traceflow - - # each with the layer the TCP header is at, Ethernet or the loopback - # header being the first - cases = [ - ('in 62 IPv4', 64, ethernet(nest4(base.TCP_SEGMENT, 62, 6))), - ('in 63 IPv4', 65, ethernet(nest4(base.TCP_SEGMENT, 63, 6))), - ('in 100 IPv4', 102, ethernet(nest4(base.TCP_SEGMENT, 100, 6))), - ('behind 31 (IPv4, AH) pairs', 64, ethernet(ah_pairs(31))), - ('behind 32 (IPv4, AH) pairs', 66, ethernet(ah_pairs(32))), - ('4in6 in 61 IPv6', 64, ethernet6(nest6(TCP4, 61, 4))), - ('4in6 in 62 IPv6', 65, ethernet6(nest6(TCP4, 62, 4))), - ('in 31 IPv4 behind 31 headers', 64, ethernet(nest4(chain(31, 6) + base.TCP_SEGMENT, 31, 60))), - ('in 31 IPv4 behind 32 headers', 65, ethernet(nest4(chain(32, 6) + base.TCP_SEGMENT, 31, 60))), - ('in 40 IPv4 behind 23 headers', 65, ethernet(nest4(chain(23, 6) + base.TCP_SEGMENT, 40, 60))), - ('RAW, in 63 IPv4', 64, unread(nest4(base.TCP_SEGMENT, 63, 6), RAW)), - ('RAW, in 64 IPv4', 65, unread(nest4(base.TCP_SEGMENT, 64, 6), RAW)), - ('NULL, in 62 IPv4', 64, unread(loopback(nest4(base.TCP_SEGMENT, 62, 6), 2, 'little'), NULL)), - ('NULL, in 63 IPv4', 65, unread(loopback(nest4(base.TCP_SEGMENT, 63, 6), 2, 'little'), NULL)), - ('IPV6, 4in6 in 62 IPv6', 64, unread(nest6(TCP4, 62, 4), IPV6)), - ('IPV6, 4in6 in 63 IPv6', 65, unread(nest6(TCP4, 63, 4), IPV6)), - ] - for count, tpid in ((61, base.C_TAG), (62, base.C_TAG), (61, base.S_TAG), (62, base.S_TAG), - (63, base.C_TAG)): - cases.append((f'behind {count} tags of {tpid:#06x}', count + 3, tags(TCP4, 0x0800, count, tpid))) - for label, layer, packet in cases: - with self.subTest(case=label), mock.patch.dict('sys.modules', stand_ins()): - data = tcp_reassembly(packet, count=1) - trace = tcp_traceflow(packet, data_link=LinkType.ETHERNET, count=1) - if layer > 64: - self.assertIsNone(data) - self.assertIsNone(trace) - self.assertEqual(self.decline(packet), []) - continue - self.assertIsNotNone(data) - self.assertIsNotNone(trace) - self.assertEqual(data.header, base.TCP_SEGMENT[:20]) - - def test_ipv4_fragments_are_read_only_within_the_frame_layer_bound(self) -> None: - from pcapkit.toolkit.pypcapfile import ipv4_reassembly - - # each with the layer the IPv4 header is at - cases = [ - ('in 62 IPv6', 64, ethernet6(nest6(FRAGMENT, 62, 4))), - ('in 63 IPv6', 65, ethernet6(nest6(FRAGMENT, 63, 4))), - ('RAW, in 63 IPv6', 64, unread(nest6(FRAGMENT, 63, 4), RAW)), - ('RAW, in 64 IPv6', 65, unread(nest6(FRAGMENT, 64, 4), RAW)), - ('LOOP, in 62 IPv6', 64, unread(loopback(nest6(FRAGMENT, 62, 4), 24, 'big'), LOOP)), - ('LOOP, in 63 IPv6', 65, unread(loopback(nest6(FRAGMENT, 63, 4), 24, 'big'), LOOP)), - ('in 61 IPv6 behind a header', 64, ethernet6(nest6(chain(1, 4) + FRAGMENT, 61, 60))), - ('in 61 IPv6 behind 2 headers', 65, ethernet6(nest6(chain(2, 4) + FRAGMENT, 61, 60))), - ] - for count, tpid in ((62, base.C_TAG), (63, base.C_TAG), (62, base.S_TAG), (63, base.S_TAG)): - cases.append((f'behind {count} tags of {tpid:#06x}', count + 2, tags(FRAGMENT, 0x0800, count, tpid))) - for label, layer, packet in cases: - with self.subTest(case=label), mock.patch.dict('sys.modules', stand_ins()): - data = ipv4_reassembly(packet, count=1) - if layer > 64: - self.assertIsNone(data) - else: - self.assertIsNotNone(data) - self.assertEqual(data.header, FRAGMENT[:20]) - - def test_tcp_over_ipv6_warns_only_within_the_frame_layer_bound(self) -> None: - # The default engine finds no TCP past the bound, so none is left out there. - cases = [ - ('behind 31 (IPv6, Hop-by-Hop) pairs', 64, ethernet6(hbh_pairs(31))), - ('behind 32 (IPv6, Hop-by-Hop) pairs', 66, ethernet6(hbh_pairs(32))), - ('behind 61 tags', 64, tags(TCP6, 0x86DD, 61, base.C_TAG)), - ('behind 62 tags', 65, tags(TCP6, 0x86DD, 62, base.C_TAG)), - ('in 40 IPv6 behind 22 headers', 64, ethernet6(nest6(chain(22, 6) + base.TCP_SEGMENT, 40, 60))), - ('in 40 IPv6 behind 23 headers', 65, ethernet6(nest6(chain(23, 6) + base.TCP_SEGMENT, 40, 60))), - ('6in4 in 61 IPv4', 64, ethernet(nest4(TCP6, 61, 41))), - ('6in4 in 62 IPv4', 65, ethernet(nest4(TCP6, 62, 41))), - ('RAW, behind 31 (IPv6, Hop-by-Hop) pairs', 63, unread(hbh_pairs(31), RAW)), - ('RAW, in 63 IPv6', 64, unread(nest6(base.TCP_SEGMENT, 63, 6), RAW)), - ('RAW, in 64 IPv6', 65, unread(nest6(base.TCP_SEGMENT, 64, 6), RAW)), - ] - for label, layer, packet in cases: - with self.subTest(case=label), mock.patch.dict('sys.modules', stand_ins()): - messages = self.decline(packet) - self.assertEqual(len(messages), int(layer <= 64), messages) - - def test_the_walks_read_nothing_past_the_bound(self) -> None: - from pcapkit.toolkit import pypcapfile - - with self.subTest(case='1000 VLAN tags'): - self.assertIsNone(pypcapfile._untag(tags(TCP4, 0x0800, 1000, base.C_TAG).packet)) - self.assertEqual(pypcapfile._untag(tags(TCP4, 0x0800, 62, base.C_TAG).packet)[2], 64) - with self.subTest(case='a chain past the bound'), mock.patch.dict('sys.modules', stand_ins()): - # the innermost IPv4 header is at layer 41, so 23 headers fit behind it - packet = ethernet(nest4(chain(32, 6) + base.TCP_SEGMENT, 40, 60)) - with mock.patch.object(pypcapfile, '_default_extension_header', - wraps=pypcapfile._default_extension_header) as parse: - self.assertIsNone(pypcapfile.tcp_reassembly(packet, count=1)) - # no header past the bound is asked of the default engine's parser - self.assertEqual(len({call.args for call in parse.call_args_list}), 23) - with self.subTest(case='the helpers other toolkits import'): - # They read a header as the first layer of its frame, as before; - # the same header deeper in a frame is read past the bound. - inner = wire.ipv4(chain(32, 6) + base.TCP_SEGMENT, proto=60) - self.assertEqual(pypcapfile._upper_layer(4, inner), (6, base.TCP_SEGMENT)) - self.assertEqual(pypcapfile._past_extension_headers(60, inner[20:]), (6, base.TCP_SEGMENT)) - self.assertEqual(pypcapfile._walk_upper_layer(4, inner, 1), (6, base.TCP_SEGMENT, 34)) - self.assertIsNone(pypcapfile._walk_upper_layer(4, inner, 41)) - self.assertIsNone(pypcapfile._walk_ipv4_payload(60, inner[20:], 41)) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/toolkit/test_pypcapfile_unit.py b/tests/toolkit/test_pypcapfile_unit.py index b49334cb60..ad643345ac 100644 --- a/tests/toolkit/test_pypcapfile_unit.py +++ b/tests/toolkit/test_pypcapfile_unit.py @@ -17,16 +17,12 @@ import ctypes import importlib.util import ipaddress -import os import struct -import tempfile import types import unittest import warnings -from tests._support import close_extractor, reimport_once_per_class -from tests.foundation import _roundtrip as wire -from tests.toolkit import test_tunnelled_tcp_flow_key_1581_unit as tunnels +from tests._support import reimport_once_per_class RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) @@ -130,10 +126,14 @@ def __init__(self, payload) -> None: def make_packet(layer, *, timestamp: int = 1511106545, timestamp_us: int = 471719, ns_resolution: bool = False, capture_len: int = 86, packet_len: int = 86, - linktype: int = 1): - """Build a stand-in for :class:`pcapfile.structs.pcap_packet`, of a capture of ``linktype``.""" + header=None): + """Build a stand-in for :class:`pcapfile.structs.pcap_packet`. + + Packets given the same ``header`` are of one capture; by default each is of its own. + + """ return types.SimpleNamespace( - header=[types.SimpleNamespace(ns_resolution=ns_resolution, ll_type=linktype)], + header=[types.SimpleNamespace(ns_resolution=ns_resolution)] if header is None else header, timestamp=timestamp, timestamp_us=timestamp_us, capture_len=capture_len, @@ -142,146 +142,20 @@ def make_packet(layer, *, timestamp: int = 1511106545, timestamp_us: int = 47171 ) -def ipv6_frame(packet: bytes) -> FakeEthernet: - """An Ethernet frame carrying IPv6, as PyPCAPFile leaves it: payload hex-encoded.""" +def ipv6_frame(nxt: int, payload: bytes = b'\x00' * 20) -> FakeEthernet: + """An Ethernet frame carrying IPv6 whose Next Header is ``nxt``, as PyPCAPFile leaves it.""" + packet = (b'\x60\x00\x00\x00' + len(payload).to_bytes(2, 'big') + bytes([nxt, 64]) + + bytes(range(32)) + payload) frame = FakeEthernet(binascii.hexlify(packet)) frame.type = 0x86DD return frame -#: A TCP segment with a full header and a payload. -TCP_SEGMENT = wire.tcp(b'payload', seq=1, syn=True) - -#: An IPv6 jumbogram carrying :data:`TCP_SEGMENT`: a Payload Length of 0, and a -#: Hop-by-Hop Options header whose Jumbo Payload option gives the real one (RFC 2675). -JUMBOGRAM = (wire.ipv6(b'', nxt=0) - + bytes([6, 0, 0xC2, 4]) + (8 + len(TCP_SEGMENT)).to_bytes(4, 'big') + TCP_SEGMENT) - -#: TPIDs of the 802.1Q customer tag and the 802.1ad service tag. -C_TAG, S_TAG = 0x8100, 0x88A8 - - -def vlan_tags(ethertype: int, *tpids: int) -> bytes: - """The octets after the first TPID of an Ethernet frame tagged once per ``tpids``. - - That is, each tag's control information and the EtherType after it -- the - next tag's TPID, or ``ethertype`` after the last tag. - - """ - return b''.join((100 + number).to_bytes(2, 'big') + kind.to_bytes(2, 'big') - for number, kind in enumerate((*tpids[1:], ethertype))) - - -def tagged(packet: bytes, ethertype: int, *tpids: int) -> bytes: - """A raw Ethernet frame carrying ``packet`` behind one VLAN tag per ``tpids``.""" - return wire.ethernet(vlan_tags(ethertype, *tpids) + packet, tpids[0]) - - -def tagged_frame(packet: bytes, ethertype: int, *tpids: int) -> FakeEthernet: - """A tagged frame as PyPCAPFile leaves it: type the first TPID, payload hex-encoded.""" - frame = FakeEthernet(binascii.hexlify(vlan_tags(ethertype, *tpids) + packet)) - frame.type = tpids[0] - return frame - - -def fake_ip_decoder() -> dict[str, types.ModuleType]: - """Stand-in modules for :mod:`pcapfile.protocols.network.ip`, for :data:`sys.modules`. - - Their ``IP`` records what it was given, refuses what is not an IPv4 header - of at least 20 octets as the real decoder does, and decodes the fields as - it does, with the options and the payload hex-encoded. - - """ - class IP: - """Stand-in for :class:`pcapfile.protocols.network.ip.IP`.""" - - calls: list = [] - - def __init__(self, packet: bytes, layers: int = 0) -> None: - self.calls.append((packet, layers)) - if packet[0] >> 4 != 4 or packet[0] & 0x0F <= 4: - raise AssertionError('not an IPv4 packet.') - fields = struct.unpack('!BBHHHBBHII', packet[:20]) - self.v, self.hl = fields[0] >> 4, fields[0] & 0x0F - self.tos, self.len, self.id = fields[1:4] - self.flags, self.off = fields[4] >> 13, fields[4] & 0x1FFF - self.ttl, self.p, self.sum = fields[5:8] - self.src, self.dst = (str(ipaddress.IPv4Address(field)).encode() for field in fields[8:]) - # the options and payload, sliced without a bounds check, as the real one does - self.opt = binascii.hexlify(packet[20:self.hl * 4]) - self.payload = binascii.hexlify(packet[self.hl * 4:]) - - modules = {name: types.ModuleType(name) for name in ( - 'pcapfile', 'pcapfile.protocols', 'pcapfile.protocols.network', - 'pcapfile.protocols.network.ip')} - modules['pcapfile.protocols.network.ip'].IP = IP - return modules - - -def data_offset(segment: bytes, words: int) -> bytes: +def with_data_offset(segment: bytes, words: int) -> bytes: """``segment``, a TCP segment, with its Data Offset set to ``words``.""" return segment[:12] + bytes([words << 4 | segment[12] & 0x0F]) + segment[13:] -def total_length(packet: bytes, value: int) -> bytes: - """``packet``, an IPv4 packet with no options, with its Total Length set to ``value``.""" - header = bytearray(packet[:20]) - header[2:4], header[10:12] = value.to_bytes(2, 'big'), b'\x00\x00' - header[10:12] = wire.checksum(bytes(header)).to_bytes(2, 'big') - return bytes(header) + packet[20:] - - -def tunnel(packet: bytes, protocol: int, **fields) -> FakeEthernet: - """An IPv4 frame tunnelling ``packet``, as PyPCAPFile decodes it: payload hex-encoded.""" - return FakeEthernet(FakeIP(binascii.hexlify(packet), p=protocol, **fields)) - - -#: IPv4 options the default engine's IPv4 parser rejects: a Record Route of -#: length 1, one whose length runs past the option area, and a Router Alert of -#: length 2. -BAD_IPV4_OPTIONS = (b'\x07\x01\x00\x00', b'\x07\x08\x00\x00', b'\x94\x02\x00\x00') - -#: IPv4 options it accepts: four No Operations, a Router Alert, and an End of -#: Option List followed by octets it leaves as padding. -GOOD_IPV4_OPTIONS = (b'\x01' * 4, b'\x94\x04\x00\x00', b'\x00\xff\xff\xff') - - -def ah(nxt: int, words: int = 4) -> bytes: - """An IP Authentication Header whose Payload Length is ``words``, then next header ``nxt``.""" - icv = b'\xaa' * max((words + 2) * 4 - 12, 0) - return bytes([nxt, words, 0, 0]) + (0x100).to_bytes(4, 'big') + (1).to_bytes(4, 'big') + icv - - -def options_header(nxt: int, padn: bytes = b'\x01\x04\x00\x00\x00\x00') -> bytes: - """An IPv6 Hop-by-Hop or Destination Options header of 8 octets, then next header ``nxt``.""" - return bytes([nxt, 0]) + padn - - -#: IPv6 extension headers the default engine reads past inside IPv4, each with -#: its protocol number, then TCP: Hop-by-Hop and Destination Options, Routing -#: (type 0, then type 2), a Fragment header (a later fragment), Mobility (a -#: Binding Refresh Request) and HIP (an I1, with no parameters). -IPV4_EXTENSION_HEADERS = ( - ('hop-by-hop', 0, options_header(6)), - ('destination options', 60, options_header(6)), - ('routing type 0', 43, bytes([6, 0, 0, 0]) + bytes(4)), - ('routing type 2', 43, bytes([6, 2, 2, 1]) + bytes(20)), - ('later fragment', 44, bytes([6, 0, 0, 0x10]) + (1).to_bytes(4, 'big')), - ('mobility', 135, bytes([6, 1, 0, 0]) + bytes(12)), - ('hip', 139, bytes([6, 4, 0x01, 0x21]) + bytes(36)), -) - - -def ihl(packet: bytes, words: int) -> bytes: - """``packet``, an IPv4 packet, with its IHL set to ``words``.""" - return bytes([0x40 | words]) + packet[1:] - - -def udp_fragment(**fields) -> bytes: - """A first IPv4 fragment of a UDP datagram, more to come.""" - return wire.ipv4(wire.udp(b'u' * 24)[:16], proto=17, mf=True, **fields) - - @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') class PyPCAPFileToolkitTests(unittest.TestCase): def setUp(self) -> None: @@ -363,6 +237,18 @@ def test_packet2dict_and_chain_tolerate_an_undecoded_frame(self) -> None: self.assertEqual(packet2chain(make_packet(bytearray(b'\x01')), data_link=LinkType.ETHERNET), 'ETHERNET:Raw') + def test_packet2dict_reports_an_undecoded_frames_octets(self) -> None: + # GH#1667: the engine hands a frame left undecoded over hex-encoded + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit.pypcapfile import packet2dict + + frame = make_ipv4(make_tcp(b'x' * 7)) + self.assertEqual(len(frame), 47) + for link in (LinkType.RAW, LinkType.LINUX_SLL, LinkType.ETHERNET): + with self.subTest(link=link): + info = packet2dict(make_packet(binascii.hexlify(frame)), data_link=link) + self.assertEqual(info[link.name], {'raw_len': 47, 'raw': frame}) + ########################################################################## # Address and hex-text parsing helpers (see GH#743). ########################################################################## @@ -523,156 +409,129 @@ def test_tcp_adapters_decline_non_tcp_and_truncated_segments(self) -> None: ('not tcp', make_packet(FakeEthernet(FakeIP(b'x' * 40, p=17)))), ('too short', make_packet(FakeEthernet(FakeIP(b'short')))), ('already decoded', make_packet(FakeEthernet(FakeIP(object())))), - # data from mid-datagram, which only looks like a TCP header (#1576) - ('later fragment', make_packet(FakeEthernet(FakeIP(make_tcp(b'x' * 20), off=185)))), ): with self.subTest(case=label): self.assertIsNone(tcp_reassembly(packet, count=1)) self.assertIsNone(tcp_traceflow(packet, data_link=LinkType.ETHERNET, count=1)) - def test_transport_takes_the_first_fragment_to_the_total_length(self) -> None: + def test_a_later_fragment_carries_no_tcp_header(self) -> None: + # GH#1576: from its offset on, a fragment's data is no TCP header + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit.pypcapfile import _transport, tcp_reassembly, tcp_traceflow + + segment = make_tcp(b'A' * 20) + later = make_packet(FakeEthernet(FakeIP(segment, off=185))) + self.assertIsNone(_transport(later.packet.payload)) + self.assertIsNone(tcp_reassembly(later, count=1)) + self.assertIsNone(tcp_traceflow(later, data_link=LinkType.ETHERNET, count=1)) + self.assertEqual(_transport(FakeIP(segment, flags=0b001)), segment) # a first fragment + + def test_the_tcp_segment_ends_at_the_total_length(self) -> None: + # GH#1577: the Ethernet padding after the datagram is no part of the segment from pcapkit.toolkit.pypcapfile import _transport segment = make_tcp(b'x') - for label, ipv4, want in ( - # #1577: octets past the Total Length, e.g. Ethernet padding, are not payload - ('padded', FakeIP(binascii.hexlify(segment + bytes(5)), len=20 + len(segment)), segment), - ('options, padded', - FakeIP(binascii.hexlify(segment + bytes(3)), hl=6, len=24 + len(segment)), segment), - ('total length 0, as TSO leaves it', - FakeIP(binascii.hexlify(segment + b'more'), len=0), segment + b'more'), - ('total length past the capture', FakeIP(binascii.hexlify(segment), len=200), segment), - ('total length shorter than the header', FakeIP(binascii.hexlify(segment), len=12), None), - ('total length cuts the tcp header short', FakeIP(binascii.hexlify(segment), len=30), None), - # #1576: a later fragment carries no TCP header, however it looks - ('last fragment', FakeIP(binascii.hexlify(segment), off=1), None), - ('later fragment, more to follow', - FakeIP(binascii.hexlify(segment), off=185, flags=0b001), None), - ('first fragment', FakeIP(binascii.hexlify(segment), flags=0b001), segment), + padded = segment + b'\x00' * 5 + for label, fields, expected in ( + ('padded', {'len': 41}, segment), + ('with options', {'len': 45, 'hl': 6, 'opt': b'\x01' * 4}, segment), + ('total length 0, as TSO leaves it (GH#1547)', {'len': 0}, padded), + ('total length short of the header', {'len': 19}, None), ): with self.subTest(case=label): - self.assertEqual(_transport(ipv4), want) + self.assertEqual(_transport(FakeIP(padded, **fields)), expected) + def test_a_data_offset_below_five_words_is_no_tcp_header(self) -> None: + # GH#1590: the header is not read as 20 octets + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit.pypcapfile import _transport, tcp_reassembly, tcp_traceflow -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class PyPCAPFileTCPOverIPv6Tests(unittest.TestCase): - """TCP over IPv6 is left out, with one warning per capture. C.f. #1513.""" + segment = make_tcp(b'data') + for words in range(5): + with self.subTest(data_offset=words): + packet = make_packet(FakeEthernet(FakeIP(with_data_offset(segment, words)))) + self.assertIsNone(_transport(packet.packet.payload)) + self.assertIsNone(tcp_reassembly(packet, count=1)) + self.assertIsNone(tcp_traceflow(packet, data_link=LinkType.ETHERNET, count=1)) + self.assertEqual(_transport(FakeIP(segment)), segment) - def setUp(self) -> None: - reimport_once_per_class(self) + def test_an_ihl_past_the_captured_octets_is_no_ipv4_layer(self) -> None: + # GH#1591: pypcapfile slices the options at the IHL, however few octets + # there are -- here up to 20 short of a declared 60 + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit.pypcapfile import _network, ipv4_reassembly, tcp_reassembly, tcp_traceflow + + for short in (1, 4, 8, 20): + with self.subTest(octets_short=short): + options = binascii.hexlify(bytes(40 - short)) + cut = make_packet(FakeEthernet(FakeIP(b'', hl=15, flags=0b001, opt=options))) + self.assertIsNone(_network(cut)) + self.assertIsNone(ipv4_reassembly(cut, count=1)) + self.assertIsNone(tcp_reassembly(cut, count=1)) + self.assertIsNone(tcp_traceflow(cut, data_link=LinkType.ETHERNET, count=1)) + + whole = FakeIP(b'', hl=15, flags=0b001, opt=binascii.hexlify(bytes(40))) + self.assertIs(_network(make_packet(FakeEthernet(whole))), whole) - def decline(self, *packets) -> list[str]: - """Feed ``packets`` to both TCP adapters, frame 1 first; return their warnings.""" + def test_tcp_over_ipv6_warns_once_per_capture(self) -> None: + # GH#1513 from pcapkit.const.reg.linktype import LinkType from pcapkit.toolkit.pypcapfile import tcp_reassembly, tcp_traceflow from pcapkit.utilities.warnings import AttributeWarning + first, second = [types.SimpleNamespace(ns_resolution=False)], [types.SimpleNamespace(ns_resolution=False)] + frames = ( + (first, ipv6_frame(17)), # UDP over IPv6 + (first, FakeEthernet(FakeIP(b'x' * 40, p=17))), + (first, ipv6_frame(6)), # TCP over IPv6: warns + (second, ipv6_frame(6)), # another capture: warns of its own + (first, ipv6_frame(6)), + (second, ipv6_frame(6)), + ) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter('always') - for count, packet in enumerate(packets, start=1): + for count, (header, frame) in enumerate(frames, 1): + packet = make_packet(frame, header=header) self.assertIsNone(tcp_reassembly(packet, count=count)) self.assertIsNone(tcp_traceflow(packet, data_link=LinkType.ETHERNET, count=count)) - return [str(item.message) for item in caught if issubclass(item.category, AttributeWarning)] - - def test_tcp_over_ipv6_warns_once_per_capture(self) -> None: - first, second = (make_packet(ipv6_frame(wire.ipv6(TCP_SEGMENT, nxt=6))) for _ in range(2)) - second.header = first.header # both frames of one capture - - with self.assertLogs('pcapkit', level='WARNING') as logs: - messages = self.decline(first, second) - self.assertEqual(len(messages), 1, messages) - self.assertTrue(messages[0].startswith('Frame 1: TCP over IPv6 is left out'), messages) - self.assertIn("'pypcapfile' has no IPv6 decoder", messages[0]) - self.assertEqual(len(logs.records), 1) - - def test_interleaved_captures_each_warn_once(self) -> None: - one, other, one_again, other_again = ( - make_packet(ipv6_frame(wire.ipv6(TCP_SEGMENT, nxt=6))) for _ in range(4)) - one_again.header, other_again.header = one.header, other.header + self.assertEqual([str(item.message) for item in caught if item.category is AttributeWarning], [ + f"Frame {count}: TCP over IPv6 is left out of TCP reassembly and flow tracing, as 'pypcapfile' " + 'has no IPv6 decoder; later such frames of this capture are left out without another warning' + for count in (3, 4)]) - messages = self.decline(one, other, one_again, other_again) - self.assertEqual([message.split(':')[0] for message in messages], ['Frame 1', 'Frame 2']) - - def test_the_oldest_capture_is_forgotten_beyond_the_limit(self) -> None: - from unittest import mock + def test_either_tcp_adapter_alone_warns_of_tcp_over_ipv6(self) -> None: + # GH#1513: a capture whose TCP over IPv6 reaches only one of the two + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit.pypcapfile import tcp_reassembly, tcp_traceflow + from pcapkit.utilities.warnings import AttributeWarning - from pcapkit.toolkit import pypcapfile as toolkit + for adapter, kwargs in ((tcp_reassembly, {}), (tcp_traceflow, {'data_link': LinkType.ETHERNET})): + with self.subTest(adapter=adapter.__name__): + header = [types.SimpleNamespace(ns_resolution=False)] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + for count in (1, 2): + self.assertIsNone(adapter(make_packet(ipv6_frame(6), header=header), count=count, + **kwargs)) + self.assertEqual([(item.category, str(item.message)[:37]) for item in caught], + [(AttributeWarning, 'Frame 1: TCP over IPv6 is left out of')]) + + def test_tcp_over_ipv6_is_read_off_a_whole_ipv6_header_only(self) -> None: + # GH#1513: the Next Header of an IPv6 frame, whose 40-octet header was captured + from pcapkit.toolkit.pypcapfile import tcp_reassembly - first, second, third, first_again, third_again = ( - make_packet(ipv6_frame(wire.ipv6(TCP_SEGMENT, nxt=6))) for _ in range(5)) - first_again.header, third_again.header = first.header, third.header - - # Two remembered at most: the third capture pushes the first out, so the - # first warns a second time -- repeated, never omitted -- while the - # third is still remembered. - with mock.patch.object(toolkit, 'TCP_WARNED_LIMIT', 2), \ - mock.patch.object(toolkit, '_tcp_warned', type(toolkit._tcp_warned)()): - messages = self.decline(first, second, third, first_again, third_again) - self.assertEqual(len(toolkit._tcp_warned), 2) - self.assertEqual([message.split(':')[0] for message in messages], - ['Frame 1', 'Frame 2', 'Frame 3', 'Frame 4']) - - def test_tcp_behind_extension_headers_warns(self) -> None: - hop_by_hop = bytes([6, 0]) + b'\x01\x04' + bytes(4) # PadN, 8 octets - long_hop_by_hop = bytes([6, 1]) + b'\x01\x0c' + bytes(12) # PadN, 16 octets - options_then_routing = bytes([43, 0]) + b'\x01\x04' + bytes(4) + bytes([6, 0]) + bytes(6) - auth = struct.pack('!BBHII', 6, 4, 0, 0x100, 1) + bytes(12) # (4 + 2) * 4 octets - mobility = bytes([6, 1]) + bytes(14) # payload proto, 16 octets - hip = bytes([6, 4, 0x01, 0x21]) + bytes(36) # next header, 40 octets - shim6 = bytes([6, 0]) + bytes(6) # next header, 8 octets - for label, packet in ( - ('hop-by-hop', wire.ipv6(hop_by_hop + TCP_SEGMENT, nxt=0)), - ('16-octet hop-by-hop', wire.ipv6(long_hop_by_hop + TCP_SEGMENT, nxt=0)), - ('destination options, routing', wire.ipv6(options_then_routing + TCP_SEGMENT, nxt=60)), - ('authentication header', wire.ipv6(auth + TCP_SEGMENT, nxt=51)), - ('first fragment', wire.ipv6_fragment(TCP_SEGMENT, mf=True, nxt=6)), - ('mobility', wire.ipv6(mobility + TCP_SEGMENT, nxt=135)), - ('hip', wire.ipv6(hip + TCP_SEGMENT, nxt=139)), - ('shim6', wire.ipv6(shim6 + TCP_SEGMENT, nxt=140)), - ('hop-by-hop, mobility', wire.ipv6(bytes([135, 0]) + bytes(6) + mobility + TCP_SEGMENT, - nxt=0)), - ('first fragment, shim6', wire.ipv6_fragment(shim6 + TCP_SEGMENT, mf=True, nxt=140)), - ): + whole = binascii.unhexlify(ipv6_frame(6).payload)[:40] + cases = [(f'{cut} octets', 0x86DD, whole[:cut], 0) for cut in (0, 6, 7, 39)] + cases += [(f'ethertype {ethertype:#06x}', ethertype, whole, 0) for ethertype in (0x0800, 0x0806, 0x8100)] + cases.append(('whole', 0x86DD, whole, 1)) + for label, ethertype, data, warns in cases: with self.subTest(case=label): - messages = self.decline(make_packet(ipv6_frame(packet))) - self.assertEqual(len(messages), 1, messages) - - def test_no_warning_without_tcp_over_ipv6(self) -> None: - arp = FakeEthernet(binascii.hexlify(b'\x00\x01\x08\x00' + bytes(24))) - arp.type = 0x0806 - decoded = ipv6_frame(b'') - decoded.payload = object() # a decoder PyPCAPFile does not have yet - for label, layer in ( - ('undecoded link', b'raw'), - ('arp', arp), - ('ipv6 payload decoded', decoded), - ('icmpv6', ipv6_frame(wire.ipv6(b'\x80\x00' + bytes(6), nxt=58))), - ('udp', ipv6_frame(wire.ipv6(wire.udp(b'x'), nxt=17))), - ('esp', ipv6_frame(wire.ipv6(b'\x00\x00\x01\x00' + bytes(4) + TCP_SEGMENT, nxt=50))), - # the header's own length runs to the end of the packet, so nothing - # follows it -- as in options-internet.pcap's HIP frames - ('hip filling the packet', ipv6_frame(wire.ipv6(bytes([6, 6, 0x01, 0x21]) + bytes(52), - nxt=139))), - ('mobility, no next header', ipv6_frame(wire.ipv6(bytes([59, 0]) + bytes(6) - + TCP_SEGMENT, nxt=135))), - ('experimental 253', ipv6_frame(wire.ipv6(bytes([6, 0]) + bytes(6) + TCP_SEGMENT, - nxt=253))), - ('truncated ipv6 header', ipv6_frame(wire.ipv6(b'', nxt=6)[:39])), - ('truncated extension header', ipv6_frame(wire.ipv6(b'\x06\x00\x01', nxt=0))), - ('truncated tcp header', ipv6_frame(wire.ipv6(TCP_SEGMENT[:19], nxt=6))), - ('extension header overruns the frame', - ipv6_frame(wire.ipv6(bytes([6, 200]) + bytes(6) + TCP_SEGMENT, nxt=0))), - # a later fragment's data is a slice of the payload, not a header, - # even when it happens to look like one - ('later fragment', ipv6_frame(wire.ipv6_fragment(TCP_SEGMENT, offset=8, nxt=6))), - ('later fragment, shim6', - ipv6_frame(wire.ipv6_fragment(bytes([6, 0]) + bytes(6) + TCP_SEGMENT, offset=8, - nxt=140))), - ('first fragment, then a later one', ipv6_frame(wire.ipv6_fragment( - struct.pack('!BBHI', 6, 0, 8, 7) + TCP_SEGMENT, mf=True, nxt=44))), - ): - with self.subTest(case=label): - self.assertEqual(self.decline(make_packet(layer)), []) + frame = FakeEthernet(binascii.hexlify(data)) + frame.type = ethertype + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + self.assertIsNone(tcp_reassembly(make_packet(frame), count=1)) + self.assertEqual(len(caught), warns, [str(item.message) for item in caught]) def test_capture_of_keys_a_ctypes_header_by_its_address(self) -> None: from pcapkit.toolkit.pypcapfile import _capture_of @@ -698,303 +557,39 @@ def test_capture_of_keys_a_ctypes_header_by_its_address(self) -> None: packet.header = value self.assertEqual(_capture_of(packet), (id(value), value)) + def test_tcp_over_ipv6_warns_once_per_capture_of_ctypes_records(self) -> None: + # GH#1513: pypcapfile's record reaches its savefile header through a + # ``ctypes`` pointer field, which builds a new pointer on every read + from pcapkit.toolkit.pypcapfile import tcp_reassembly -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -class PyPCAPFileEncapsulatedTCPTests(unittest.TestCase): - """TCP behind VLAN tags or in IP tunnels is read or warned about. C.f. #1537.""" - - decline = PyPCAPFileTCPOverIPv6Tests.decline - - def setUp(self) -> None: - reimport_once_per_class(self) - - def test_vlan_tagged_tcp_over_ipv6_warns(self) -> None: - hop_by_hop = bytes([6, 0]) + b'\x01\x04' + bytes(4) - for label, frame in ( - ('802.1Q', tagged_frame(wire.ipv6(TCP_SEGMENT, nxt=6), 0x86DD, C_TAG)), - ('802.1ad', tagged_frame(wire.ipv6(TCP_SEGMENT, nxt=6), 0x86DD, S_TAG)), - ('QinQ', tagged_frame(wire.ipv6(TCP_SEGMENT, nxt=6), 0x86DD, S_TAG, C_TAG)), - ('two customer tags', tagged_frame(wire.ipv6(TCP_SEGMENT, nxt=6), 0x86DD, C_TAG, C_TAG)), - ('802.1Q, hop-by-hop', tagged_frame(wire.ipv6(hop_by_hop + TCP_SEGMENT, nxt=0), 0x86DD, - C_TAG)), - ): - with self.subTest(case=label): - messages = self.decline(make_packet(frame)) - self.assertEqual(len(messages), 1, messages) - self.assertTrue(messages[0].startswith('Frame 1: TCP over IPv6 is left out'), - messages) - - def test_tunnelled_tcp_over_ipv6_warns(self) -> None: - # C.f. #1581: TCP whose innermost IP layer is IPv6 is TCP over IPv6, - # however deep the tunnel, and warns as such. - from unittest import mock - - tcp4, tcp6 = wire.ipv4(TCP_SEGMENT, proto=6), wire.ipv6(TCP_SEGMENT, nxt=6) - hop_by_hop = bytes([6, 0]) + b'\x01\x04' + bytes(4) - for label, frame in ( - ('6in4', tunnel(tcp6, 41)), - ('6in6', ipv6_frame(wire.ipv6(tcp6, nxt=41))), - ('6in4in6', ipv6_frame(wire.ipv6(wire.ipv4(tcp6, proto=41), nxt=4))), - ('6in4in4', tunnel(wire.ipv4(tcp6, proto=41), 4)), - ('6in6 in a 4in4', tunnel(wire.ipv4(wire.ipv6(tcp6, nxt=41), proto=41), 4)), - ('outer first fragment', tunnel(tcp6, 41, flags=0b001)), - ('inner hop-by-hop', tunnel(wire.ipv6(hop_by_hop + TCP_SEGMENT, nxt=0), 41)), - ('tunnel behind hop-by-hop', - ipv6_frame(wire.ipv6(bytes([41, 0]) + b'\x01\x04' + bytes(4) + tcp6, nxt=0))), - ('6in6 behind 802.1Q', tagged_frame(wire.ipv6(tcp6, nxt=41), 0x86DD, C_TAG)), - ('6in4 behind ah', tunnel(ah(41) + tcp6, 51)), - ('inner ipv6 jumbogram', tunnel(JUMBOGRAM, 41)), - ('outer ipv4 option', tunnel(tcp6, 41, hl=6, opt=b'\x94\x04\x00\x00')), # router alert - ): - with self.subTest(case=label), mock.patch.dict('sys.modules', fake_ip_decoder()): - messages = self.decline(make_packet(frame)) - self.assertEqual(len(messages), 1, messages) - self.assertTrue(messages[0].startswith('Frame 1: TCP over IPv6 is left out'), messages) - - def test_tunnelled_tcp_over_ipv4_is_found_in_the_innermost_ipv4(self) -> None: - # C.f. #1581: TCP whose innermost IP layer is IPv4 is read from that - # layer, which keys it, rather than left out with a warning. - from unittest import mock - - from pcapkit.toolkit.pypcapfile import _innermost, _transport - - tcp4 = wire.ipv4(TCP_SEGMENT, proto=6) - options = make_tcp(b'x', options=b'\x02\x04\x05\xb4') - ipv4_options = make_ipv4(TCP_SEGMENT, options=b'\x01\x01\x01\x00') - for label, frame, inner, segment in ( - ('4in4', tunnel(tcp4, 4), tcp4, TCP_SEGMENT), - ('4in6', ipv6_frame(wire.ipv6(tcp4, nxt=4)), tcp4, TCP_SEGMENT), - ('4in4in4', tunnel(wire.ipv4(tcp4, proto=4), 4), tcp4, TCP_SEGMENT), - ('4in6in4', tunnel(wire.ipv6(tcp4, nxt=4), 41), tcp4, TCP_SEGMENT), - ('4in6 behind QinQ', tagged_frame(wire.ipv6(tcp4, nxt=4), 0x86DD, S_TAG, C_TAG), tcp4, - TCP_SEGMENT), - ('4in4 behind ah', tunnel(ah(4) + tcp4, 51), tcp4, TCP_SEGMENT), - ('4in4 behind destination options', tunnel(options_header(4) + tcp4, 60), tcp4, TCP_SEGMENT), - ('outer first fragment', tunnel(tcp4, 4, flags=0b001), tcp4, TCP_SEGMENT), - ('outer padded', tunnel(tcp4 + bytes(6), 4, len=20 + len(tcp4)), tcp4, TCP_SEGMENT), - ('outer ipv4 option', tunnel(tcp4, 4, hl=6, opt=b'\x94\x04\x00\x00'), tcp4, TCP_SEGMENT), - ('inner ipv4 options', tunnel(ipv4_options, 4), ipv4_options, TCP_SEGMENT), - ('inner first fragment', tunnel(wire.ipv4(TCP_SEGMENT, proto=6, mf=True), 4), - wire.ipv4(TCP_SEGMENT, proto=6, mf=True), TCP_SEGMENT), - ('inner total length 0', tunnel(total_length(tcp4, 0), 4), total_length(tcp4, 0), - TCP_SEGMENT), - ('inner ah', tunnel(wire.ipv4(ah(6) + TCP_SEGMENT, proto=51), 4), - wire.ipv4(ah(6) + TCP_SEGMENT, proto=51), TCP_SEGMENT), - ('tcp options', tunnel(wire.ipv4(options, proto=6), 4), wire.ipv4(options, proto=6), options), - # the default engine reads these out of the raw octets (#1518), in a tunnel too - ('tcp option of length 0', tunnel(wire.ipv4(make_tcp(b'x', options=b'\x02\x00\x05\xb4'), - proto=6), 4), - wire.ipv4(make_tcp(b'x', options=b'\x02\x00\x05\xb4'), proto=6), - make_tcp(b'x', options=b'\x02\x00\x05\xb4')), - ('data offset past the capture', tunnel(wire.ipv4(data_offset(TCP_SEGMENT, 15), proto=6), 4), - wire.ipv4(data_offset(TCP_SEGMENT, 15), proto=6), data_offset(TCP_SEGMENT, 15)), - ): - with self.subTest(case=label), mock.patch.dict('sys.modules', fake_ip_decoder()) as modules: - IP = modules['pcapfile.protocols.network.ip'].IP - ipv4, upper, depth = _innermost(make_packet(frame)) - self.assertIsInstance(ipv4, IP) - self.assertIsNone(upper) - self.assertEqual(IP.calls[-1], (inner, 0)) - self.assertEqual(_transport(ipv4, depth), segment) - - def test_no_warning_where_the_default_engine_finds_no_tcp(self) -> None: - from unittest import mock - - tcp4, tcp6 = wire.ipv4(TCP_SEGMENT, proto=6), wire.ipv6(TCP_SEGMENT, nxt=6) - arp = b'\x00\x01\x08\x00' + bytes(24) - decoded = tagged_frame(b'', 0x86DD, C_TAG) - decoded.payload = object() # a decoder PyPCAPFile does not have yet - cut_short = FakeEthernet(binascii.hexlify(b'\x00\x64\x86')) # three octets of a tag - cut_short.type = C_TAG - for label, frame in ( - ('not tunnelled', FakeEthernet(FakeIP(b'x' * 40, p=17))), - ('tunnelled udp', tunnel(wire.ipv4(wire.udp(b'x'), proto=17), 4)), - ('gre', tunnel(bytes(4) + tcp4, 47)), - ('ipv4 payload decoded', FakeEthernet(FakeIP(object(), p=4))), - # a later fragment's data is a slice of the payload, not a header - ('outer later fragment', tunnel(tcp4, 4, off=3)), - ('inner later fragment', tunnel(wire.ipv4(TCP_SEGMENT, proto=6, offset=8), 4)), - ('inner ipv6 later fragment', tunnel(wire.ipv6_fragment(TCP_SEGMENT, offset=8, nxt=6), 41)), - ('inner not version 4', tunnel(b'\x65' + tcp4[1:], 4)), - ('inner header under 20 octets', tunnel(b'\x44' + tcp4[1:], 4)), - ('inner header overruns the packet', tunnel(b'\x4f' + tcp4[1:], 4)), - ('inner ipv4 truncated', tunnel(tcp4[:19], 4)), - ('inner ipv6 truncated', tunnel(tcp6[:39], 41)), - ('inner tcp truncated', tunnel(wire.ipv4(TCP_SEGMENT[:19], proto=6), 4)), - # the Total Length ends the packet inside the TCP header, though more was captured - ('inner total length cuts the tcp header short', tunnel(total_length(tcp4, 30), 4)), - ('outer total length cuts the tunnel short', tunnel(tcp4, 4, len=40)), - ('inner payload length cuts the tcp header short', - tunnel(tcp6[:4] + (10).to_bytes(2, 'big') + tcp6[6:], 41)), - # a Payload Length of 0 with no Jumbo Payload option leaves no payload - ('inner payload length 0, no jumbo option', tunnel(tcp6[:4] + b'\x00\x00' + tcp6[6:], 41)), - ('ipv6 payload length 0, no jumbo option', ipv6_frame(tcp6[:4] + b'\x00\x00' + tcp6[6:])), - ('tunnelled data offset 4', tunnel(wire.ipv4(data_offset(TCP_SEGMENT, 4), proto=6), 4)), - ('tunnelled ipv6 data offset 4', tunnel(wire.ipv6(data_offset(TCP_SEGMENT, 4), nxt=6), 41)), - # nor is a tunnel whose outer IPv4 header the default engine rejects (#1591, #1596) - ('outer ipv4 option', tunnel(tcp4, 4, hl=6, opt=b'\x94\x02\x00\x00')), # router alert, length 2 - ('outer ipv4 header overruns the packet', tunnel(tcp4, 4, hl=6, opt=b'\x94\x04')), - # or the inner one (#1581), as the default engine dissects nothing past it - ('inner ipv4 option', tunnel(make_ipv4(TCP_SEGMENT, options=b'\x94\x02\x00\x00'), 4)), - ('6in4 in an inner ipv4 it rejects', - tunnel(make_ipv4(tcp6, protocol=41, options=b'\x94\x02\x00\x00'), 4)), - # nor is one whose Data Offset is under 5 words, over IPv6 alone too - ('ipv6 data offset 4', ipv6_frame(wire.ipv6(data_offset(TCP_SEGMENT, 4), nxt=6))), - ('802.1Q, ipv6 data offset 0', tagged_frame(wire.ipv6(data_offset(TCP_SEGMENT, 0), nxt=6), - 0x86DD, C_TAG)), - ('inner esp', ipv6_frame(wire.ipv6(wire.ipv6(bytes(8) + TCP_SEGMENT, nxt=50), nxt=41))), - ('tunnel header overruns the frame', - ipv6_frame(wire.ipv6(bytes([41, 200]) + bytes(6) + tcp6, nxt=0))), - ('802.1Q, then arp', tagged_frame(arp, 0x0806, C_TAG)), - ('802.1Q, payload decoded', decoded), - ('802.1Q cut short', cut_short), - # 0x9100 is a pre-standard QinQ TPID, which the default engine does not dissect - ('0x9100 tag', tagged_frame(tcp6, 0x86DD, 0x9100)), - ): - with self.subTest(case=label), mock.patch.dict('sys.modules', fake_ip_decoder()): - self.assertEqual(self.decline(make_packet(frame)), []) - - def test_network_decodes_the_ipv4_behind_the_tags(self) -> None: - from unittest import mock - - from pcapkit.toolkit.pypcapfile import _network - - packet = wire.ipv4(TCP_SEGMENT, proto=6) - with mock.patch.dict('sys.modules', fake_ip_decoder()) as modules: - IP = modules['pcapfile.protocols.network.ip'].IP - ipv4 = _network(make_packet(tagged_frame(packet, 0x0800, S_TAG, C_TAG))) - self.assertIsInstance(ipv4, IP) - self.assertEqual(IP.calls, [(packet, 0)]) # every tag stripped, network layer only - - def test_malformed_ipv4_behind_tags_warns_once_per_frame(self) -> None: - # As PyPCAPFile._decode warns of an untagged frame it cannot decode, - # however many adapters then fetch the tagged frame's network layer. - from unittest import mock - - from pcapkit.const.reg.linktype import LinkType - from pcapkit.toolkit.pypcapfile import ipv4_reassembly, tcp_reassembly, tcp_traceflow - from pcapkit.utilities.warnings import AttributeWarning - - bad = tagged_frame(b'\x44' + wire.ipv4(TCP_SEGMENT, proto=6)[1:], 0x0800, C_TAG) # IHL 4 - other = FakeEthernet(FakeIP(b'x' * 40, p=17, flags=0b010)) # decoded, nothing to warn of - packets = [make_packet(frame) for frame in (bad, bad, other, bad)] - for packet in packets[1:]: - packet.header = packets[0].header # one capture - with mock.patch.dict('sys.modules', fake_ip_decoder()), \ - warnings.catch_warnings(record=True) as caught: - warnings.simplefilter('always') - for count, packet in enumerate(packets, start=1): - self.assertIsNone(ipv4_reassembly(packet, count=count)) - self.assertIsNone(tcp_reassembly(packet, count=count)) - self.assertIsNone(tcp_traceflow(packet, data_link=LinkType.ETHERNET, count=count)) - messages = [str(item.message) for item in caught if issubclass(item.category, AttributeWarning)] - self.assertEqual(messages, [ - f"Frame {count}: decoding failed (AssertionError('not an IPv4 " - "packet.')); frame left undecoded" for count in (1, 2, 4) - ]) - - def test_tcp_over_ipv6_the_tcp_parser_rejects_still_warns(self) -> None: - # Over IPv6 the default engine reads such a header out of the raw octets - # its TCP parser leaves, in a tunnel too, where that IPv6 layer keys it (#1581). - for label, frame in ( - ('data offset past the capture', ipv6_frame(wire.ipv6(data_offset(TCP_SEGMENT, 9), nxt=6))), - ('option of length 0', ipv6_frame(wire.ipv6(make_tcp(b'x', options=b'\x02\x00\x05\xb4'), - nxt=6))), - ('jumbogram', ipv6_frame(JUMBOGRAM)), - ('6in4, data offset past the capture', tunnel(wire.ipv6(data_offset(TCP_SEGMENT, 9), nxt=6), 41)), - ('6in6, option past its area', - ipv6_frame(wire.ipv6(wire.ipv6(make_tcp(b'x', options=b'\x08\x0a\x00\x00'), nxt=6), nxt=41))), - ): - with self.subTest(case=label): - messages = self.decline(make_packet(frame)) - self.assertEqual(len(messages), 1, messages) - self.assertTrue(messages[0].startswith('Frame 1: TCP over IPv6'), messages) - - def test_detection_leaves_the_warnings_filters_and_the_logger_alone(self) -> None: - # The warnings filters and the logger are process-wide state of the - # caller's: telling a frame apart must neither change them nor reset - # what they have already seen. - import logging - - from pcapkit.const.reg.linktype import LinkType - from pcapkit.toolkit.pypcapfile import tcp_reassembly, tcp_traceflow - - logger = logging.getLogger('pcapkit') - level = logger.level - # options the default engine's TCP and IPv4 parsers each reject, so log - frames = (ipv6_frame(wire.ipv6(wire.ipv6(data_offset(TCP_SEGMENT, 9), nxt=6), nxt=41)), - FakeEthernet(FakeIP(ah(6) + make_tcp(b'x', options=b'\x02\x00\x05\xb4'), p=51)), - FakeEthernet(FakeIP(TCP_SEGMENT, hl=6, opt=b'\x94\x02\x00\x00')), - ipv6_frame(wire.ipv6(TCP_SEGMENT, nxt=6))) + class Header(ctypes.Structure): + _fields_ = [('ns_resolution', ctypes.c_bool)] - def user_warning() -> None: - warnings.warn('a warning of the caller', UserWarning) + class Record(ctypes.Structure): # as pcapfile.structs.pcap_packet + _fields_ = [('header', ctypes.POINTER(Header))] + first, second = Header(), Header() with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter('default') - filters = list(warnings.filters) - user_warning() - # the adapters called as the engine calls them, with nothing between - for count, frame in enumerate(frames, start=1): - packet = make_packet(frame) - tcp_reassembly(packet, count=count) - tcp_traceflow(packet, data_link=LinkType.ETHERNET, count=count) - self.assertEqual(warnings.filters, filters) - self.assertEqual(logger.level, level) - user_warning() # the same call site, so the 'default' action shows it only once - self.assertEqual([str(item.message) for item in caught if item.category is UserWarning], - ['a warning of the caller']) - - def test_each_frame_is_walked_once_per_adapter_call(self) -> None: - # Declining costs one header walk per frame and adapter, however the - # frame is malformed: nothing is retried or dissected again. - from unittest import mock + warnings.simplefilter('always') + for count, header in enumerate((first, first, second, first, second), 1): + record = Record(ctypes.pointer(header)) + record.packet = ipv6_frame(6) + self.assertIsNone(tcp_reassembly(record, count=count)) + self.assertEqual([str(item.message)[:8] for item in caught], ['Frame 1:', 'Frame 3:']) - from pcapkit.const.reg.linktype import LinkType - from pcapkit.protocols.protocol import ProtocolBase + def test_the_captures_warned_of_tcp_over_ipv6_are_bounded(self) -> None: from pcapkit.toolkit import pypcapfile as toolkit - built = [] # type: list[str] - init = ProtocolBase.__init__ - - def spy_init(this, *args, **kwargs): - built.append(type(this).__name__) - return init(this, *args, **kwargs) - - cut = wire.ipv6(wire.ipv6(make_tcp(b'', options=b'\x02\x04\x05\xb4'), nxt=6), nxt=41)[:-4] - packets = [make_packet(ipv6_frame(cut)) for _ in range(10)] # a TCP header cut short in a tunnel - for packet in packets[1:]: - packet.header = packets[0].header # one capture, which warns once - with mock.patch.object(toolkit, '_walk_upper_layer', wraps=toolkit._walk_upper_layer) as walk, \ - mock.patch.object(ProtocolBase, '__init__', spy_init), \ - warnings.catch_warnings(record=True) as caught: + self.assertEqual(toolkit.IPV6_TCP_WARNED_LIMIT, 32) + headers = [[types.SimpleNamespace(ns_resolution=False)] for _ in range(33)] + with warnings.catch_warnings(record=True) as caught: warnings.simplefilter('always') - for count, packet in enumerate(packets, start=1): - self.assertIsNone(toolkit.tcp_reassembly(packet, count=count)) - self.assertIsNone(toolkit.tcp_traceflow(packet, data_link=LinkType.ETHERNET, count=count)) - self.assertEqual([str(item.message).split(': ', 1)[0] for item in caught], ['Frame 1']) - # the outer IPv6 header and the inner one, per frame and adapter ... - self.assertEqual(walk.call_count, 2 * 2 * len(packets)) - # ... and no protocol of the default engine's dissected along the way - self.assertEqual(built, []) - - def test_tcp_over_ipv6_warns_once_per_capture_tunnelled_or_not(self) -> None: - # C.f. #1581: TCP over IPv6 in a tunnel is no kind of its own. - from unittest import mock - - frames = (ipv6_frame(wire.ipv6(TCP_SEGMENT, nxt=6)), - ipv6_frame(wire.ipv6(wire.ipv6(TCP_SEGMENT, nxt=6), nxt=41)), - tagged_frame(wire.ipv6(TCP_SEGMENT, nxt=6), 0x86DD, C_TAG), - tunnel(wire.ipv6(TCP_SEGMENT, nxt=6), 41), - tunnel(wire.ipv4(wire.ipv6(TCP_SEGMENT, nxt=6), proto=41), 4)) - packets = [make_packet(frame) for frame in frames] - for packet in packets[1:]: - packet.header = packets[0].header # all five frames of one capture - - with mock.patch.dict('sys.modules', fake_ip_decoder()): - messages = self.decline(*packets) - self.assertEqual(len(messages), 1, messages) - self.assertTrue(messages[0].startswith('Frame 1: TCP over IPv6'), messages) + for header in headers: + toolkit.tcp_reassembly(make_packet(ipv6_frame(6), header=header), count=1) + toolkit.tcp_reassembly(make_packet(ipv6_frame(6), header=headers[0]), count=2) # forgotten + toolkit.tcp_reassembly(make_packet(ipv6_frame(6), header=headers[-1]), count=3) # remembered + self.assertEqual([str(item.message)[:8] for item in caught], ['Frame 1:'] * 33 + ['Frame 2:']) + self.assertEqual(len(toolkit._ipv6_tcp_warned), 32) # pylint: disable=protected-access @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') @@ -1065,6 +660,8 @@ def test_tcp_traceflow_reports_the_flow_endpoints(self) -> None: data = tcp_traceflow(packet, data_link=LinkType.ETHERNET, count=5) self.assertIsNotNone(data) self.assertEqual(data.protocol, LinkType.ETHERNET) + self.assertEqual(data.header, segment[:20]) + self.assertEqual(bytes(data.payload), b'body') self.assertEqual(data.index, 5) self.assertTrue(data.syn) self.assertFalse(data.fin) @@ -1075,6 +672,31 @@ def test_tcp_traceflow_reports_the_flow_endpoints(self) -> None: self.assertEqual(data.timestamp, 1511106545.471719) self.assertEqual(data.frame['ETHERNET']['IP']['src'], '10.1.1.2') + def test_the_adapters_own_checks_hold_on_the_real_decoder(self) -> None: + from pcapfile.protocols.network.ip import IP + + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit.pypcapfile import ipv4_reassembly, tcp_reassembly, tcp_traceflow + + def packet(raw: bytes): + return make_packet(FakeEthernet(IP(raw))) + + segment = make_tcp(b'x', flags=0b00011000) + # GH#1577: a datagram padded to the Ethernet minimum + padded = packet(make_ipv4(segment) + b'\x00' * 5) + data = tcp_reassembly(padded, count=1) + self.assertEqual((data.header, bytes(data.payload), data.len), (segment[:20], b'x', 1)) + self.assertEqual(bytes(tcp_traceflow(padded, data_link=LinkType.ETHERNET, count=1).payload), b'x') + # GH#1576, GH#1590: a later fragment, and a Data Offset of 4 + for label, raw in (('later fragment', make_ipv4(b'A' * 40, offset=185)), + ('data offset 4', make_ipv4(with_data_offset(segment, 4)))): + with self.subTest(case=label): + self.assertIsNone(tcp_reassembly(packet(raw), count=2)) + self.assertIsNone(tcp_traceflow(packet(raw), data_link=LinkType.ETHERNET, count=2)) + # GH#1591: an IHL of 15 in a 40-octet packet + cut = bytes([0x4F]) + make_ipv4(bytes(20), protocol=17, flags=0b001)[1:] + self.assertIsNone(ipv4_reassembly(packet(cut), count=3)) + def test_ipv4_2dict_unhexes_the_options_field(self) -> None: from pcapfile.protocols.network.ip import IP @@ -1139,446 +761,6 @@ def test_ethernet2dict_reports_the_default_engines_mac_format(self) -> None: self.assertEqual(info['dst'], '01:00:5e:01:03:03') self.assertEqual(info['src'], '00:0c:29:3f:1a:07') - def test_engine_warns_once_per_capture_on_tcp_over_ipv6(self) -> None: - from pcapkit import extract - - # One IPv4 TCP frame, then two IPv6 TCP frames and an IPv6 UDP one. The - # engine builds a new ``pcap_packet`` per frame, so this also checks that - # the real savefile header keys one capture -- see ``_capture_of``. - frames = ( - wire.ethernet(wire.ipv4(TCP_SEGMENT, proto=6), 0x0800), - wire.ethernet(wire.ipv6(TCP_SEGMENT, nxt=6), 0x86DD), - wire.ethernet(wire.ipv6(wire.tcp(b'more', seq=8, ack=1), nxt=6), 0x86DD), - wire.ethernet(wire.ipv6(wire.udp(b'x'), nxt=17), 0x86DD), - ) - with tempfile.TemporaryDirectory() as tmp: - path = os.path.join(tmp, 'mixed.pcap') - with open(path, 'wb') as file: - file.write(wire.pcap([(1, number, frame) for number, frame in enumerate(frames)])) - - for read in ('first', 'second'): # reading it again is another capture - with self.subTest(read=read), warnings.catch_warnings(record=True) as caught: - warnings.simplefilter('always') - extractor = extract(fin=path, nofile=True, engine='pypcapfile', - reassembly=True, tcp=True, trace=True, - trace_fout=os.path.join(tmp, read), trace_format='json') - try: - messages = [str(item.message) for item in caught - if 'TCP over IPv6' in str(item.message)] - self.assertEqual(extractor._exnam, 'pypcapfile') - self.assertEqual(len(messages), 1, messages) - self.assertTrue(messages[0].startswith('Frame 2: '), messages) - # ...while the IPv4 frame is still reassembled and traced - self.assertEqual([tuple(flow.index) for flow in extractor.trace.tcp], - [(1,)]) - self.assertEqual(len(extractor.reassembly.tcp), 1) - finally: - close_extractor(extractor) - - def test_network_decodes_behind_vlan_tags_as_untagged(self) -> None: - from pcapfile.protocols.linklayer.ethernet import Ethernet - - from pcapkit.toolkit.pypcapfile import _ipv4_2dict, _maybe_unhex, _network - - packet = make_ipv4(make_tcp(b'body'), options=b'\x94\x04\x00\x00') - untagged = Ethernet(wire.ethernet(packet, 0x0800), 1).payload - want = _ipv4_2dict(untagged), _maybe_unhex(bytes(untagged.payload)) - for label, tpids in (('802.1Q', (C_TAG,)), ('802.1ad', (S_TAG,)), ('QinQ', (S_TAG, C_TAG)), - ('two customer tags', (C_TAG, C_TAG))): - with self.subTest(case=label): - ipv4 = _network(make_packet(Ethernet(tagged(packet, 0x0800, *tpids), 1))) - self.assertEqual(type(ipv4).__name__, 'IP') - self.assertEqual((_ipv4_2dict(ipv4), _maybe_unhex(bytes(ipv4.payload))), want) - - for label, frame, undecoded in ( - ('802.1Q, then ipv6', tagged(wire.ipv6(TCP_SEGMENT, nxt=6), 0x86DD, C_TAG), False), - ('802.1Q, then not ipv4', tagged(b'\x65' + packet[1:], 0x0800, C_TAG), True), - ('802.1Q, then a truncated ipv4 header', tagged(packet[:19], 0x0800, C_TAG), True), - ('802.1Q cut short', wire.ethernet(b'\x00\x64\x08', C_TAG), False), - ('0x9100 tag', tagged(packet, 0x0800, 0x9100), False), - ): - with self.subTest(case=label), warnings.catch_warnings(record=True) as caught: - warnings.simplefilter('always') - self.assertIsNone(_network(make_packet(Ethernet(frame, 1)), 7)) - self.assertEqual([str(item.message).split(' decoding failed')[0] for item in caught], - ['Frame 7: '] if undecoded else []) - - def test_engine_warns_alike_of_tagged_and_untagged_ipv4_it_cannot_decode(self) -> None: - from pcapkit import extract - - bad = b'\x44' + wire.ipv4(TCP_SEGMENT, proto=6)[1:] # IHL 4 - frames = (wire.ethernet(bad, 0x0800), tagged(bad, 0x0800, S_TAG, C_TAG), - wire.ethernet(wire.ipv4(TCP_SEGMENT, proto=6), 0x0800)) - with tempfile.TemporaryDirectory() as tmp: - path = os.path.join(tmp, 'undecodable.pcap') - with open(path, 'wb') as file: - file.write(wire.pcap([(1, number, frame) for number, frame in enumerate(frames)])) - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter('always') - extractor = extract(fin=path, nofile=True, engine='pypcapfile', reassembly=True, - ipv4=True, tcp=True, trace=True, - trace_fout=os.path.join(tmp, 'trace'), trace_format='json') - try: - self.assertEqual(extractor._exnam, 'pypcapfile') - messages = [str(item.message) for item in caught if 'decoding failed' in str(item.message)] - # one each, the untagged from the engine and the tagged from the toolkit, - # in the same words after the frame number - self.assertEqual([message.split(': ', 1)[0] for message in messages], - ['Frame 1', 'Frame 2']) - self.assertEqual(messages[0].split(': ', 1)[1], messages[1].split(': ', 1)[1]) - self.assertEqual([tuple(flow.index) for flow in extractor.trace.tcp], [(3,)]) - finally: - close_extractor(extractor) - - #: What :meth:`read_both` compares between the two engines. - ASPECTS = ('ipv4', 'tcp', 'trace', 'ipv4 datagrams', 'tcp datagrams', 'flows') - - def read(self, engine: str, path: str, tracedir: str) -> dict: - """Every field ``engine`` hands IPv4 and TCP reassembly and TCP flow tracing for ``path``. - - Inputs are keyed by frame number, and a trace input leaves out ``frame``, - the engine's own frame object -- as in ``test_engine_agreement_runtime``. - - """ - from unittest import mock - - from pcapkit import extract - from pcapkit.foundation.reassembly.reassembly import ReassemblyBase - from pcapkit.foundation.traceflow.traceflow import TraceFlowBase - - seen = {'ipv4': {}, 'tcp': {}, 'trace': {}} # type: dict - reassemble, trace = ReassemblyBase.__call__, TraceFlowBase.__call__ - - def fields(packet, skip=()): - return {key: bytes(value) if isinstance(value, bytearray) else value - for key, value in packet.items() if key not in skip} - - def spy_reassembly(this, packet): - seen[type(this).__name__.lower()][packet.num] = fields(packet) - return reassemble(this, packet) - - def spy_trace(this, packet): - seen['trace'][packet.index] = fields(packet, ('frame',)) - return trace(this, packet) - - with warnings.catch_warnings(record=True) as caught, \ - mock.patch.object(ReassemblyBase, '__call__', spy_reassembly), \ - mock.patch.object(TraceFlowBase, '__call__', spy_trace): - warnings.simplefilter('always') - extractor = extract(fin=path, nofile=True, engine=engine, reassembly=True, - ipv4=True, ipv6=False, tcp=True, trace=True, - trace_fout=tracedir, trace_format='json') - try: - self.assertEqual(extractor._exnam, engine) - self.assertEqual([str(item.message) for item in caught - if 'left out' in str(item.message)], []) - for kind in ('ipv4', 'tcp'): - seen[f'{kind} datagrams'] = [fields(datagram, ('packet',)) - for datagram in getattr(extractor.reassembly, kind)] - seen['flows'] = [(flow.label, tuple(flow.index)) for flow in extractor.trace.tcp] - finally: - close_extractor(extractor) - return seen - - def read_both(self, frames) -> tuple[dict, dict]: - """:meth:`read` a capture of ``frames`` with ``pypcapfile``, then with ``default``.""" - with tempfile.TemporaryDirectory() as tmp: - path = os.path.join(tmp, 'capture.pcap') - with open(path, 'wb') as file: - file.write(wire.pcap([(1, number, frame) for number, frame in enumerate(frames)])) - return (self.read('pypcapfile', path, os.path.join(tmp, 'pypcapfile')), - self.read('default', path, os.path.join(tmp, 'default'))) - - def test_vlan_tagged_ipv4_agrees_with_the_default_engine(self) -> None: - datagram = wire.udp(b'u' * 24) - frames = ( - tagged(wire.ipv4(wire.tcp(b'hello', seq=100, syn=True), proto=6), 0x0800, C_TAG), - tagged(wire.ipv4(wire.tcp(b'ok', seq=500, ack=106, sport=9, dport=40000), - proto=6, reverse=True), 0x0800, C_TAG), - tagged(wire.ipv4(wire.tcp(b'world', seq=106, ack=501), proto=6), 0x0800, S_TAG, C_TAG), - tagged(wire.ipv4(wire.tcp(b'!', seq=111, ack=501), proto=6), 0x0800, C_TAG, C_TAG), - tagged(wire.ipv4(wire.tcp(b'bye', seq=112, ack=501, fin=True), proto=6), 0x0800, S_TAG), - wire.ethernet(wire.ipv4(wire.tcp(b'late', seq=115, ack=501), proto=6), 0x0800), - # an IPv4 datagram in two tagged fragments, for IPv4 reassembly - tagged(wire.ipv4(datagram[:16], proto=17, mf=True, ident=0x99), 0x0800, C_TAG), - tagged(wire.ipv4(datagram[16:], proto=17, offset=16, ident=0x99), 0x0800, S_TAG, C_TAG), - # IPv4 and TCP options, on a flow of their own - tagged(make_ipv4(make_tcp(b'opts', options=b'\x02\x04\x05\xb4'), - options=b'\x94\x04\x00\x00'), 0x0800, C_TAG), - ) - mine, theirs = self.read_both(frames) - - self.assertEqual(sorted(mine['tcp']), [1, 2, 3, 4, 5, 6, 9]) - self.assertEqual(sorted(mine['ipv4']), [1, 2, 3, 4, 5, 6, 7, 8, 9]) - self.assertIn(datagram, [bytes(item['payload']) for item in mine['ipv4 datagrams']]) - self.assertEqual(len(mine['flows']), 2) - for aspect in self.ASPECTS: - with self.subTest(aspect=aspect): - self.assertEqual(mine[aspect], theirs[aspect]) - - def test_later_fragment_is_not_read_as_tcp(self) -> None: - # C.f. #1576: a SYN, then later fragments whose data merely looks like a - # TCP header -- untagged and tagged -- and a first fragment, which does - # carry one. - data = b'A' * 20 + b'B' * 20 - frames = ( - wire.ethernet(wire.ipv4(wire.tcp(b'', seq=0, syn=True), proto=6, df=True), 0x0800), - wire.ethernet(wire.ipv4(data, proto=6, offset=1480, ident=9), 0x0800), - tagged(wire.ipv4(data, proto=6, offset=8, mf=True, ident=10), 0x0800, C_TAG), - wire.ethernet(wire.ipv4(wire.tcp(b'first', seq=1, ack=1), proto=6, mf=True, ident=11), - 0x0800), - ) - mine, theirs = self.read_both(frames) - - self.assertEqual(sorted(mine['tcp']), [1, 4]) - for aspect in self.ASPECTS: - with self.subTest(aspect=aspect): - self.assertEqual(mine[aspect], theirs[aspect]) - - def test_tcp_payload_ends_at_the_total_length(self) -> None: - # C.f. #1577: Ethernet padding past the Total Length is not TCP payload. - frames = ( - wire.ethernet(wire.ipv4(wire.tcp(b'x', seq=100, ack=1), proto=6), 0x0800) + bytes(5), - tagged(wire.ipv4(wire.tcp(b'y', seq=101, ack=1), proto=6), 0x0800, C_TAG) + bytes(1), - # 0, as TCP segmentation offload leaves it: the rest of the frame - wire.ethernet(total_length(wire.ipv4(wire.tcp(b'tso', seq=102, ack=1), proto=6), 0), - 0x0800), - # shorter than the header: no payload at all - wire.ethernet(total_length(wire.ipv4(wire.tcp(b'bogus', seq=105, ack=1), proto=6), 12), - 0x0800), - # past the end of the capture: what was captured - wire.ethernet(total_length(wire.ipv4(wire.tcp(b'cut', seq=105, ack=1), proto=6), 200), - 0x0800), - ) - mine, theirs = self.read_both(frames) - - self.assertEqual(sorted(mine['tcp']), [1, 2, 3, 5]) - self.assertEqual([mine['tcp'][number]['payload'] for number in (1, 2, 3, 5)], - [b'x', b'y', b'tso', b'cut']) - for aspect in self.ASPECTS: - with self.subTest(aspect=aspect): - self.assertEqual(mine[aspect], theirs[aspect]) - - def test_engine_warns_of_tunnelled_tcp_over_ipv6_only(self) -> None: - # C.f. #1581: tunnelled TCP is read where its innermost IP layer is - # IPv4, keyed by that layer, and is TCP over IPv6 where it is IPv6. - from pcapkit import extract - - syn = make_tcp(b'', flags=0x02, options=b'\x02\x04\x05\xb4') - frames = ( - tunnels.frame(tunnels.ip(4, TCP_SEGMENT, '10.9.9.1', '10.9.9.2', 6)), - tunnels.tunnelled('4in4', TCP_SEGMENT), - # a tunnelled TCP header with a Data Offset under 5 words is no TCP - tunnels.tunnelled('4in4', data_offset(TCP_SEGMENT, 4)), - tunnels.tunnelled('6in4', TCP_SEGMENT), - # nor warned of again, as for TCP over IPv6 with no tunnel - wire.ethernet(wire.ipv6(TCP_SEGMENT, nxt=6), 0x86DD), - # a tunnelled SYN whose options the snapshot length cut off - tunnels.tunnelled('6in6', syn)[:-2], - tagged(tunnels.ip(4, tunnels.ip(4, TCP_SEGMENT, '10.0.0.3', '10.0.0.4', 6), - *tunnels.OUTER[4], 4), 0x0800, C_TAG), - ) - with tempfile.TemporaryDirectory() as tmp: - path = os.path.join(tmp, 'tunnels.pcap') - with open(path, 'wb') as file: - file.write(wire.pcap([(1, number, frame) for number, frame in enumerate(frames)])) - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter('always') - extractor = extract(fin=path, nofile=True, engine='pypcapfile', - reassembly=True, tcp=True, trace=True, - trace_fout=os.path.join(tmp, 'trace'), trace_format='json') - try: - messages = [str(item.message) for item in caught - if 'left out' in str(item.message)] - self.assertEqual(extractor._exnam, 'pypcapfile') - self.assertEqual(len(messages), 1, messages) - self.assertTrue(messages[0].startswith('Frame 4: TCP over IPv6'), messages) - self.assertEqual([(flow.label.rsplit('-', 1)[0], tuple(flow.index)) - for flow in extractor.trace.tcp], - [('10.9.9.1_40000-10.9.9.2_9', (1,)), ('10.0.0.1_40000-10.0.0.2_9', (2,)), - ('10.0.0.3_40000-10.0.0.4_9', (7,))]) - finally: - close_extractor(extractor) - - def test_two_connections_through_one_tunnel_are_two_flows(self) -> None: - # C.f. #1581: as for the other engines (see - # tests.toolkit.test_tunnelled_tcp_flow_key_1581_unit), over IPv4 only. - for kind in ('4in4', '4in6'): - with self.subTest(kind=kind): - mine, theirs = self.read_both(tunnels.connections(kind)) - self.assertEqual([index for _, index in mine['flows']], [(1, 2, 3), (4, 5, 6)]) - self.assertEqual([label.rsplit('-', 1)[0] for label, _ in mine['flows']], - ['10.0.0.1_40000-10.0.0.2_9', '10.0.0.3_40000-10.0.0.4_9']) - self.assertEqual(sorted({str(item['id'].src[0]) for item in mine['tcp datagrams']}), - ['10.0.0.1', '10.0.0.3']) - for aspect in self.ASPECTS: - self.assertEqual(mine[aspect], theirs[aspect], aspect) - - -@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') -@unittest.skipUnless(HAS_PYPCAPFILE, 'pypcapfile not installed or not importable') -class PyPCAPFileHeaderCheckAgreementTests(unittest.TestCase): - """Frames whose headers PyPCAPFile reads but does not check, against the default engine. - - C.f. #1590 (Data Offset below 5), #1591 (IHL past the packet), #1592 (IPv4 in - IPv6), #1593 (TCP behind AH over IPv4), #1596 (IPv4 options) and #1597 (TCP - behind IPv6 extension headers in IPv4). - - """ - - ASPECTS = PyPCAPFileToolkitAgainstRealDecodersTests.ASPECTS - read = PyPCAPFileToolkitAgainstRealDecodersTests.read - read_both = PyPCAPFileToolkitAgainstRealDecodersTests.read_both - - def setUp(self) -> None: - reimport_once_per_class(self) - - def agree(self, frames, **want) -> dict: - """:meth:`read_both` ``frames``; check each aspect agrees, and pypcapfile's inputs are ``want``.""" - mine, theirs = self.read_both(frames) - for kind, numbers in want.items(): - self.assertEqual(sorted(mine[kind]), numbers, kind) - for aspect in self.ASPECTS: - with self.subTest(aspect=aspect): - self.assertEqual(mine[aspect], theirs[aspect]) - return mine - - def test_data_offset_below_5_agrees(self) -> None: - segment = wire.tcp(b'data', seq=1) - self.agree([ - wire.ethernet(wire.ipv4(data_offset(segment, 4), proto=6), 0x0800), - tagged(wire.ipv4(data_offset(segment, 0), proto=6), 0x0800, C_TAG), - wire.ethernet(wire.ipv4(segment, proto=6), 0x0800), - ], tcp=[3], trace=[3]) - - def test_ipv4_header_past_the_packet_agrees(self) -> None: - self.agree([ - wire.ethernet(ihl(wire.ipv4(wire.tcp(b'', seq=1), proto=6), 15), 0x0800), - tagged(ihl(wire.ipv4(wire.tcp(b'', seq=1), proto=6), 15), 0x0800, C_TAG), - wire.ethernet(ihl(wire.ipv4(b'', proto=17), 6), 0x0800), - wire.ethernet(wire.ipv4(b'', proto=17), 0x0800), - ], ipv4=[4], tcp=[]) - - def test_ipv4_in_ipv6_agrees(self) -> None: - datagram = wire.udp(b'u' * 24) - first = wire.ipv4(datagram[:16], proto=17, mf=True, ident=5) - frames = [ - # one datagram, in two fragments, each in IPv6 - wire.ethernet(wire.ipv6(first, nxt=4), 0x86DD), - tagged(wire.ipv6(wire.ipv4(datagram[16:], proto=17, offset=16, ident=5), nxt=4), - 0x86DD, C_TAG), - wire.ethernet(wire.ipv6(bytes([4, 0, 1, 4, 0, 0, 0, 0]) + udp_fragment(), nxt=0), 0x86DD), - wire.ethernet(wire.ipv6(wire.ipv6(udp_fragment(), nxt=4), nxt=41), 0x86DD), - wire.ethernet(wire.ipv6(udp_fragment(), nxt=4), 0x86DD) + bytes(9), - wire.ethernet(wire.ipv6(make_ipv4(b'u' * 8, protocol=17, flags=1, - options=b'\x94\x04\x00\x00'), nxt=4), 0x86DD), - # none: DF set, a later IPv6 fragment, an IHL past the packet, options it rejects - wire.ethernet(wire.ipv6(wire.ipv4(wire.udp(b'x'), proto=17, df=True), nxt=4), 0x86DD), - wire.ethernet(wire.ipv6_fragment(udp_fragment(), offset=8, nxt=4), 0x86DD), - wire.ethernet(wire.ipv6(ihl(udp_fragment(), 15), nxt=4), 0x86DD), - wire.ethernet(wire.ipv6(make_ipv4(b'u' * 8, protocol=17, flags=1, - options=BAD_IPV4_OPTIONS[0]), nxt=4), 0x86DD), - ] - mine = self.agree(frames, ipv4=[1, 2, 3, 4, 5, 6]) - self.assertIn(datagram, [item['payload'] for item in mine['ipv4 datagrams'] if item['completed']]) - - def test_tcp_behind_ah_agrees(self) -> None: - def over_ah(segment: bytes, header: bytes = ah(6), **fields) -> bytes: - return wire.ethernet(wire.ipv4(header + segment, proto=51, **fields), 0x0800) - - frames = [ - # a flow behind AH, then behind two, tagged, and with Ethernet padding - over_ah(wire.tcp(b'', seq=0, syn=True)), - over_ah(wire.tcp(b'hello', seq=1, ack=1), ah(51) + ah(6, 1)), - tagged(wire.ipv4(ah(6) + wire.tcp(b'!', seq=6, ack=1), proto=51), 0x0800, C_TAG), - over_ah(wire.tcp(b'', seq=7, ack=1, fin=True)) + bytes(7), - over_ah(make_tcp(b'opts', options=b'\x02\x04\x05\xb4')), - # none: a Payload Length of 0, a Data Offset past the capture or below - # 5, an option the TCP parser rejects, a later fragment - over_ah(wire.tcp(b'x', seq=0), ah(6, 0)), - over_ah(data_offset(wire.tcp(b'x', seq=0), 9)), - over_ah(data_offset(wire.tcp(b'x', seq=0), 4)), - over_ah(make_tcp(b'x', options=b'\x02\x03\x00\x00')), - over_ah(wire.tcp(b'x', seq=0), offset=8), - ] - mine = self.agree(frames, tcp=[1, 2, 3, 4, 5], trace=[1, 2, 3, 4, 5]) - self.assertEqual(mine['tcp'][2]['payload'], b'hello') - - def test_tcp_behind_ipv6_extension_headers_in_ipv4_agrees(self) -> None: - # C.f. #1597: the default engine reads IPv4's protocol as IPv6 reads a - # Next Header, so dissects these headers there too, and the TCP after them. - def over(protocol: int, header: bytes, segment: bytes) -> bytes: - return wire.ethernet(wire.ipv4(header + segment, proto=protocol), 0x0800) - - frames = [over(protocol, header, wire.tcp(b'data', seq=number, sport=1000 + number)) - for number, (_, protocol, header) in enumerate(IPV4_EXTENSION_HEADERS)] - frames += [ - # two, then AH after one, tagged - over(0, options_header(60) + options_header(6), wire.tcp(b'data', seq=7)), - over(60, options_header(51) + ah(6), wire.tcp(b'data', seq=8)), - tagged(wire.ipv4(options_header(6) + wire.tcp(b'data', seq=9), proto=60), 0x0800, C_TAG), - # none: an option the default engine rejects, a header cut short, a HIP - # Header Length under 4, Shim6, and a TCP option behind it the TCP parser rejects - over(60, options_header(6, b'\x01\x09' + bytes(4)), wire.tcp(b'data', seq=10)), - wire.ethernet(wire.ipv4(options_header(6)[:4], proto=60), 0x0800), - wire.ethernet(wire.ipv4(bytes([6, 0, 0, 0]), proto=43), 0x0800), - over(139, bytes([6, 3, 0x01, 0x21]) + bytes(28), wire.tcp(b'data', seq=11)), - over(140, bytes([6, 0, 0x80, 0]) + bytes(4), wire.tcp(b'data', seq=12)), - over(60, options_header(6), make_tcp(b'x', options=b'\x02\x03\x00\x00')), - ] - self.agree(frames, tcp=list(range(1, 11)), trace=list(range(1, 11))) - - def test_ipv4_options_agree(self) -> None: - frames = [] - for options in (*GOOD_IPV4_OPTIONS, *BAD_IPV4_OPTIONS): - segment = make_tcp(b'data', src_port=1000 + len(frames)) - frames += [ - wire.ethernet(make_ipv4(segment, options=options), 0x0800), - tagged(make_ipv4(segment, options=options), 0x0800, C_TAG), - wire.ethernet(make_ipv4(b'x' * 16, protocol=17, flags=1, options=options), 0x0800), - ] - self.agree(frames, ipv4=list(range(1, 10)), tcp=[1, 2, 4, 5, 7, 8]) - - def test_tunnelled_tcp_over_ipv4_agrees(self) -> None: - # C.f. #1581: read from, and keyed by, the innermost IPv4 layer. - ip, inner, outer = tunnels.ip, tunnels.INNER[4], tunnels.OUTER - - def in4(packet: bytes, protocol: int = 4, **fields) -> bytes: - return tunnels.frame(ip(4, packet, *outer[4], protocol, **fields)) - - def tcp4(segment: bytes, connection: int = 0, protocol: int = 6, **fields) -> bytes: - return ip(4, segment, *inner[connection], protocol, **fields) - - segment = wire.tcp(b'hello', seq=1, ack=1) - bad_option = make_tcp(b'x', options=b'\x02\x00\x05\xb4') # an MSS of length 0 - frames = [ - tunnels.tunnelled('4in4', segment), - tunnels.tunnelled('4in6', segment, 1), - in4(ip(4, tcp4(segment), '172.16.0.1', '172.16.0.2', 4)), - in4(ip(6, tcp4(segment, 1), 'fd01::1', 'fd01::2', 4), 41), - in4(ah(4) + tcp4(segment), 51), - in4(options_header(4) + tcp4(segment, 1), 60), - in4(tcp4(ah(6) + segment), 4), - in4(tcp4(options_header(6) + segment, 1, 0), 4), - in4(tcp4(segment), 4, mf=True), - in4(tcp4(segment, 1, mf=True), 4), - in4(tcp4(segment)) + bytes(6), - tagged(ip(4, tcp4(segment, 1), *outer[4], 4), 0x0800, C_TAG), - # a TCP header the TCP parser rejects, read out of the raw octets (#1518) - in4(tcp4(data_offset(segment, 9))), - in4(tcp4(bad_option, 1)), - # 15 to 19: no TCP -- an inner header the IPv4 parser rejects, a later - # inner fragment, a Data Offset under 5, and behind AH, no fall-back - # to the raw octets - in4(make_ipv4(segment, options=b'\x94\x02\x00\x00')), - in4(tcp4(segment, offset=8)), - in4(tcp4(data_offset(segment, 4))), - in4(tcp4(ah(6) + bad_option, 1, 51)), - in4(tcp4(ah(6) + data_offset(segment, 9), 0, 51)), - ] - self.agree(frames, tcp=list(range(1, 15)), trace=list(range(1, 15))) - if __name__ == '__main__': unittest.main() diff --git a/tests/toolkit/test_tunnelled_tcp_flow_key_1581_unit.py b/tests/toolkit/test_tunnelled_tcp_flow_key_1581_unit.py index 443b72d5bc..4886d17811 100644 --- a/tests/toolkit/test_tunnelled_tcp_flow_key_1581_unit.py +++ b/tests/toolkit/test_tunnelled_tcp_flow_key_1581_unit.py @@ -10,8 +10,7 @@ Every capture is built here, so the module is in the unit tier. pyshark runs only where the interpreter supports it, with tshark on the path. The -``pypcapfile`` engine's counterparts need `pypcapfile`, so live in -:mod:`tests.toolkit.test_pypcapfile_unit`, which the PyPCAPFile CI cell runs. +``pypcapfile`` engine decodes no tunnel, so has no counterpart (#1708). """ from __future__ import annotations